def _perform(self):
# parameters
# image sections for each amp
bsec, dsec, tsec, direc = self.action.args.map_ccd
namps = len(bsec)
# header keyword to update
key = 'OSCANTRM'
keycom = 'Overscan trimmed?'
# get output image dimensions
max_sec = max(tsec)
# create new blank image
new = np.zeros((max_sec[1] + 1, max_sec[3] + 1), dtype=np.float32)
# loop over amps
for ia in range(namps):
# input range indices
yi0 = dsec[ia][0]
yi1 = dsec[ia][1] + 1
xi0 = dsec[ia][2]
xi1 = dsec[ia][3] + 1
# output range indices
yo0 = tsec[ia][0]
yo1 = tsec[ia][1] + 1
xo0 = tsec[ia][2]
xo1 = tsec[ia][3] + 1
# transfer to new image
new[yo0:yo1, xo0:xo1] = self.action.args.ccddata.data[yi0:yi1,
xi0:xi1]
# update amp section
sec = "[%d:" % (xo0 + 1)
sec += "%d," % xo1
sec += "%d:" % (yo0 + 1)
sec += "%d]" % yo1
self.action.args.ccddata.header['ATSEC%d' % (ia + 1)] = sec
# remove obsolete sections
self.action.args.ccddata.header.pop('ASEC%d' % (ia + 1))
self.action.args.ccddata.header.pop('BSEC%d' % (ia + 1))
self.action.args.ccddata.header.pop('DSEC%d' % (ia + 1))
self.action.args.ccddata.header.pop('CSEC%d' % (ia + 1))
# update with new image
self.action.args.ccddata.data = new
self.action.args.ccddata.header['NAXIS1'] = max_sec[3] + 1
self.action.args.ccddata.header['NAXIS2'] = max_sec[1] + 1
self.action.args.ccddata.header[key] = (True, keycom)
log_string = TrimOverscan.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
if self.config.instrument.saveintims:
kcwi_fits_writer(
self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="trim")
return self.action.args
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation
"""
method = 'average'
suffix = self.action.args.stack_type.lower()
self.logger.info("Stacking flats using method %s" % method)
combine_list = list(self.combine_list['OFNAME'])
# get flat stack output name
stname = combine_list[0].split('.fits')[0] + '_' + suffix + '.fits'
stack = []
stackf = []
for flat in combine_list:
# get flat intensity (int) image file name in redux directory
stackf.append(flat.split('.fits')[0] + '_intd.fits')
flatfn = os.path.join(self.action.args.in_directory, stackf[-1])
# using [0] gets just the image data
stack.append(kcwi_fits_reader(flatfn)[0])
stacked = ccdproc.combine(stack,
method=method,
sigma_clip=True,
sigma_clip_low_thresh=None,
sigma_clip_high_thresh=2.0)
stacked.header['IMTYPE'] = self.action.args.stack_type
stacked.header['NSTACK'] = (len(combine_list),
'number of images stacked')
stacked.header['STCKMETH'] = (method, 'method used for stacking')
for ii, fname in enumerate(stackf):
stacked.header['STACKF%d' % (ii + 1)] = (fname, "stack input file")
log_string = StackFlats.__module__
stacked.header['HISTORY'] = log_string
# output stacked flat
kcwi_fits_writer(stacked,
output_file=stname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(
frame=stacked, suffix=suffix, newtype=self.action.args.stack_type)
self.context.proctab.write_proctab()
self.logger.info(log_string)
return self.action.args
def _perform(self):
# Determine which radial velocity correction to make
correction_mode = self.config.instrument.radial_velocity_correction
options = ["none", "barycentric", "heliocentric"]
# If the config file has an invalid option, return
if not bool([el for el in options if el in correction_mode]):
self.logger.error(
'Bad config option for radial_velocity_correction\
, options are ["none", "heliocentric", "barycentric"]')
return self.action.args
suffix = 'icube' # Can be ammended to handle ocube files
obj = self.locate_object_file(suffix)
if "none" in correction_mode:
self.logger.info("Skipping radial velocity correction")
else:
self.logger.info(f"Performing {correction_mode} correction")
obj = self.heliocentric(obj, correction_mode)
if self.config.instrument.air_to_vacuum:
self.logger.info("Performing Air to Vacuum Conversion")
obj = self.air2vac(obj)
log_string = WavelengthCorrections.__module__
obj.header['HISTORY'] = log_string
kcwi_fits_writer(obj,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix=f'{suffix}w')
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix=f'_{suffix}w',
filename=self.action.args.name)
self.context.proctab.write_proctab()
# Unsure here: Is this right? it seems to make DAR happy
self.action.args.ccddata = obj
return self.action.args
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation
"""
args = self.action.args
method = 'average'
suffix = args.new_type.lower()
combine_list = list(self.combine_list['filename'])
# get master dark output name
mdname = strip_fname(combine_list[0]) + '_' + suffix + '.fits'
stack = []
stackf = []
for dark in combine_list:
# get dark intensity (int) image file name in redux directory
stackf.append(dark.split('.fits')[0] + '_int.fits')
darkfn = os.path.join(args.in_directory, stackf[-1])
# using [0] gets just the image data
stack.append(kcwi_fits_reader(darkfn)[0])
stacked = ccdproc.combine(stack, method=method, sigma_clip=True,
sigma_clip_low_thresh=None,
sigma_clip_high_thresh=2.0)
stacked.unit = stack[0].unit
stacked.header.IMTYPE = args.new_type
stacked.header['NSTACK'] = (len(combine_list),
'number of images stacked')
stacked.header['STCKMETH'] = (method, 'method used for stacking')
for ii, fname in enumerate(stackf):
stacked.header['STACKF%d' % (ii + 1)] = (fname, "stack input file")
log_string = MakeMasterDark.__module__
stacked.header['HISTORY'] = log_string
self.logger.info(log_string)
kcwi_fits_writer(stacked, output_file=mdname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=stacked, suffix=suffix,
newtype=args.new_type,
filename=self.action.args.name)
self.context.proctab.write_proctab()
return self.action.args
def _perform(self):
# Header keyword to update
key = 'GAINCOR'
keycom = 'Gain corrected?'
# print(self.action.args.ccddata.header)
number_of_amplifiers = self.action.args.namps
for amplifier in range(number_of_amplifiers):
# get amp section
section = self.action.args.ccddata.header['ATSEC%d' %
(amplifier + 1)]
parsed_section, read_forward = parse_imsec(section)
# get gain for this amp
gain = self.action.args.ccddata.header['GAIN%d' % (amplifier + 1)]
self.logger.info(
"Applying gain correction of %.3f in section %s" %
(gain, self.action.args.ccddata.header['ATSEC%d' %
(amplifier + 1)]))
self.action.args.ccddata.data[
parsed_section[0]:(parsed_section[1]+1),
parsed_section[2]:(parsed_section[3]+1)] *= gain
self.action.args.ccddata.header[key] = (True, keycom)
self.action.args.ccddata.header['BUNIT'] = ('electron', 'Pixel units')
self.action.args.ccddata.unit = 'electron'
log_string = CorrectGain.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
if self.config.instrument.saveintims:
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="gain")
return self.action.args
def _perform(self):
# TODO: implement parameter options from kcwi_stage1.pro
self.logger.info("Finding and masking cosmic rays")
# Header keyword to update
key = 'CRCLEAN'
keycom = 'cosmic rays cleaned?'
header = self.action.args.ccddata.header
if header['XPOSURE'] >= self.config.instrument.CRR_MINEXPTIME:
namps = header['NVIDINP']
read_noise = 0.
for ia in range(namps):
if 'BIASRN%d' % (ia + 1) in header:
read_noise += header['BIASRN%d' % (ia + 1)]
elif 'OSCNRN%d' % (ia + 1) in header:
read_noise += header['OSCNRN%d' % (ia + 1)]
else:
read_noise += 3.
read_noise /= float(namps)
# Set sigclip according to image parameters
sigclip = self.config.instrument.CRR_SIGCLIP
if 'FLATLAMP' in self.action.args.ccddata.header['IMTYPE']:
if self.action.args.nasmask:
sigclip = 10.
else:
sigclip = 7.
if 'OBJECT' in self.action.args.ccddata.header['IMTYPE']:
if self.action.args.ccddata.header['TTIME'] < 300.:
sigclip = 10.
mask, clean = detect_cosmics(
self.action.args.ccddata.data,
gain=1.0,
readnoise=read_noise,
psffwhm=self.config.instrument.CRR_PSFFWHM,
sigclip=sigclip,
sigfrac=self.config.instrument.CRR_SIGFRAC,
objlim=self.config.instrument.CRR_OBJLIM,
fsmode=self.config.instrument.CRR_FSMODE,
psfmodel=self.config.instrument.CRR_PSFMODEL,
verbose=self.config.instrument.CRR_VERBOSE,
sepmed=self.config.instrument.CRR_SEPMED,
cleantype=self.config.instrument.CRR_CLEANTYPE)
self.logger.info("Astroscrappy: cleaned cosmic rays")
header['history'] = "Astroscrappy: cleaned cosmic rays"
header['history'] = \
"Astroscrappy params: sigclip=%5.2f sigfrac=%5.2f " \
"objlim=%5.2f" % (
self.config.instrument.CRR_SIGCLIP,
self.config.instrument.CRR_SIGFRAC,
self.config.instrument.CRR_OBJLIM)
header['history'] = \
"Astroscrappy params: fsmode=%s psfmodel=%s psffwhm=%5.2f" % (
self.config.instrument.CRR_FSMODE,
self.config.instrument.CRR_PSFMODEL,
self.config.instrument.CRR_PSFFWHM)
header[
'history'] = "Astroscrappy params: sepmed=%s minexptime=%f" % (
self.config.instrument.CRR_SEPMED,
self.config.instrument.CRR_MINEXPTIME)
# header['history'] = "LA CosmicX run on %s" % time.strftime("%c")
# update arrays
mask = np.cast["bool"](mask)
fmask = np.where(mask)
try:
self.action.args.ccddata.flags[fmask] += 4
except AttributeError:
self.logger.warning("Flags array not found!")
n_crs = mask.sum()
self.action.args.ccddata.mask += mask
self.action.args.ccddata.data = clean
# update header
header[key] = (True, keycom)
header['NCRCLEAN'] = (n_crs, "number of cosmic ray pixels")
else:
self.logger.info("Astroscrappy: exptime < minexptime=%.1f" %
self.config.instrument.CRR_MINEXPTIME)
header['history'] = \
"Astroscrappy: exptime < minexptime=%.1f" % \
self.config.instrument.CRR_MINEXPTIME
header[key] = (False, keycom)
header['NCRCLEAN'] = (0, "number of cosmic ray pixels")
log_string = RemoveCosmicRays.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
if self.config.instrument.saveintims:
kcwi_fits_writer(
self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="crr")
return self.action.args
def _perform(self):
# Header keyword to update
key = 'FLATCOR'
keycom = 'flat corrected?'
# obj, sky
obj = None
sky = None
self.logger.info("Correcting Illumination")
if self.action.args.master_flat:
mflat = kcwi_fits_reader(
os.path.join(os.path.dirname(self.action.args.name),
self.config.instrument.output_directory,
self.action.args.master_flat))[0]
# do the correction
self.action.args.ccddata.data *= mflat.data
# update header keywords
self.action.args.ccddata.header[key] = (True, keycom)
self.action.args.ccddata.header['MFFILE'] = (
self.action.args.master_flat, "Master flat filename")
# check for obj, sky observations
if self.action.args.nasmask and self.action.args.numopen > 1:
ofn = self.action.args.ccddata.header['OFNAME']
objfn = ofn.split('.')[0] + '_obj.fits'
full_path = os.path.join(
os.path.dirname(self.action.args.name),
self.config.instrument.output_directory, objfn)
if os.path.exists(full_path):
obj = kcwi_fits_reader(full_path)[0]
# correction
obj.data *= mflat.data
# update header
obj.header[key] = (True, keycom)
obj.header['MFFILE'] = (self.action.args.master_flat,
'Master flat filename')
else:
obj = None
skyfn = ofn.split('.')[0] + '_sky.fits'
full_path = os.path.join(
os.path.dirname(self.action.args.name),
self.config.instrument.output_directory, skyfn)
if os.path.exists(full_path):
sky = kcwi_fits_reader(full_path)[0]
# correction
sky.data *= mflat.data
# update header
sky.header[key] = (True, keycom)
sky.header['MFFILE'] = (self.action.args.master_flat,
'Master flat filename')
else:
sky = None
else:
self.logger.error("No master flat found, "
"cannot correct illumination.")
self.action.args.ccddata.header[key] = (False, keycom)
log_string = CorrectIllumination.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
# write out intf image
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="intf")
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix="intf")
self.context.proctab.write_proctab()
# check for obj, sky images
if obj is not None:
kcwi_fits_writer(
obj,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="objf")
if sky is not None:
kcwi_fits_writer(
sky,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="skyf")
self.logger.info(log_string)
return self.action.args
def _perform(self):
self.logger.info("Correcting detector defects")
# Header keyword to update
key = 'BPCLEAN'
keycom = 'cleaned bad pixels?'
# Create flags for bad columns fixed
flags = np.zeros(self.action.args.ccddata.data.shape, dtype=np.uint8)
# Nod and Shuffle?
if self.action.args.nasmask and self.action.args.numopen > 1:
nastr = "_nas"
else:
nastr = ""
# Does the defect file exist?
path = "data/defect_%s_%dx%d%s.dat" % (self.action.args.ampmode.strip(),
self.action.args.xbinsize,
self.action.args.ybinsize, nastr)
package = __name__.split('.')[0]
full_path = pkg_resources.resource_filename(package, path)
number_of_bad_pixels = 0 # count of defective pixels cleaned
if os.path.exists(full_path):
self.logger.info("Reading defect list in: %s" % full_path)
defect_table = pd.read_csv(full_path, sep=r'\s+')
# range of pixels for calculating good value
pixel_range_for_good_value = 5
for index, row in defect_table.iterrows():
# Get coords and adjust for python zero bias
x0 = row['X0'] - 1
x1 = row['X1']
y0 = row['Y0'] - 1
y1 = row['Y1']
# Loop over y range
for by in range(y0, y1):
# sample on low side of bad area
values = list(self.action.args.ccddata.data[by,
x0-pixel_range_for_good_value:x0])
# sample on high side
values.extend(self.action.args.ccddata.data[by,
x1+1:x1+pixel_range_for_good_value+1])
# get replacement value
good_values = np.nanmedian(np.asarray(values))
# Replace baddies with gval
for bx in range(x0, x1):
self.action.args.ccddata.data[by, bx] = good_values
flags[by, bx] += 2
number_of_bad_pixels += 1
self.action.args.ccddata.header[key] = (True, keycom)
self.action.args.ccddata.header['BPFILE'] = (path, 'defect list')
else:
self.logger.error("Defect list not found for %s" % full_path)
self.action.args.ccddata.header[key] = (False, keycom)
self.logger.info("Cleaned %d bad pixels" % number_of_bad_pixels)
self.action.args.ccddata.header['NBPCLEAN'] = \
(number_of_bad_pixels, 'number of bad pixels cleaned')
log_string = CorrectDefects.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
# add flags array
self.action.args.ccddata.mask = flags
self.action.args.ccddata.flags = flags
if self.config.instrument.saveintims:
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="def")
return self.action.args
def _perform(self):
self.logger.info("Subtracting nod-and shuffle sky background")
# Header keyword to update
key = 'NASSUB'
keycom = 'Nod-and-shuffle subtraction done?'
target_type = 'SKY'
# get header values
ofn = self.action.args.ccddata.header['OFNAME']
shrows = self.action.args.ccddata.header['SHUFROWS']
nshfup = self.action.args.ccddata.header['NSHFUP']
nshfdn = self.action.args.ccddata.header['NSHFDN']
# units
u_out = self.action.args.ccddata.unit
# nominal conditions (sky on bottom, object in middle)
skyrow0 = 0
skyrow1 = shrows - 1
objrow0 = shrows
objrow1 = shrows + shrows - 1
# aborted script with inverted panels (sky in middle, object above)
if nshfdn != nshfup + 1:
skyrow0 = shrows
skyrow1 = shrows + shrows - 1
objrow0 = skyrow1
objrow1 = objrow0 + shrows - 1
# check limits
if (skyrow1 - skyrow0) != (objrow1 - objrow0):
self.logger.error("Nod-and-shuffle row limits error")
return self.action.args
# create intermediate images and headers
sky = self.action.args.ccddata.data.copy()
obj = self.action.args.ccddata.data.copy()
std = self.action.args.ccddata.uncertainty.array.copy()
msk = self.action.args.ccddata.mask.copy()
flg = self.action.args.ccddata.flags.copy()
skyhdr = self.action.args.ccddata.header.copy()
objhdr = self.action.args.ccddata.header.copy()
# nominal condition
if skyrow0 < 10:
self.logger.info("standard nod-and-shuffle configuration")
skystd = self.action.args.ccddata.uncertainty.array.copy()
skymsk = self.action.args.ccddata.mask.copy()
skyflg = self.action.args.ccddata.flags.copy()
# move sky to object position
sky[objrow0:objrow1, :] = obj[skyrow0:skyrow1, :]
skystd[objrow0:objrow1, :] = std[skyrow0:skyrow1, :]
skymsk[objrow0:objrow1, :] = msk[skyrow0:skyrow1, :]
skyflg[objrow0:objrow1, :] = flg[skyrow0:skyrow1, :]
# do subtraction
self.action.args.ccddata.data -= sky
self.action.args.ccddata.uncertainty.array = np.sqrt(std**2 +
skystd**2)
self.action.args.ccddata.mask += skymsk
self.action.args.ccddata.flags |= skyflg
# clean images
self.action.args.ccddata.data[skyrow0:skyrow1, :] = 0.
self.action.args.ccddata.data[(objrow1 + 1):-1, :] = 0.
self.action.args.ccddata.uncertainty.array[skyrow0:skyrow1, :] = 0.
self.action.args.ccddata.uncertainty.array[(objrow1 +
1):-1, :] = 0.
self.action.args.ccddata.mask[skyrow0:skyrow1, :] = 1
self.action.args.ccddata.mask[(objrow1 + 1):-1, :] = 1
self.action.args.ccddata.flags[skyrow0:skyrow1, :] = 64
self.action.args.ccddata.flags[(objrow1 + 1):-1, :] = 64
sky[skyrow0:skyrow1, :] = 0.
sky[(objrow1 + 1), :] = 0.
obj[skyrow0:skyrow1, :] = 0.
obj[(objrow1 + 1), :] = 0.
else:
self.logger.warning("non-standard nod-and-shuffle configuration")
skyscl = -1.
while skyscl < 0.:
if self.config.instrument.plot_level >= 2:
q = input("Enter scale factor for sky to match obj "
"(float): ")
try:
skyscl = float(q)
except ValueError:
self.logger.warning("Invalid input: %s, try again" % q)
skyscl = -1.0
else:
skyscl = 1.0
self.logger.info("Sky scaling used = %.2f" % skyscl)
objstd = self.action.args.ccddata.uncertainty.array.copy()
objmsk = self.action.args.ccddata.mask.copy()
objflg = self.action.args.ccddata.flags.copy()
# move object to sky position
obj[skyrow0:skyrow1, :] = obj[objrow0:objrow1, :]
objstd[skyrow0:skyrow1, :] = std[objrow0:objrow1, :]
objmsk[skyrow0:skyrow1, :] = msk[objrow0:objrow1, :]
objflg[skyrow0:skyrow1, :] = flg[objrow0:objrow1, :]
# do subtraction
sky *= skyscl
self.action.args.ccddata.data = obj - sky
self.action.args.ccddata.uncertainty.array = np.sqrt(std**2 +
objstd**2)
self.action.args.ccddata.mask += objmsk
self.action.args.ccddata.flags |= objflg
# clean images
self.action.args.ccddata.data[objrow0:objrow1, :] = 0.
self.action.args.ccddata.data[0:skyrow0, :] = 0.
self.action.args.ccddata.uncertainty.array[objrow0:objrow1, :] = 0.
self.action.args.ccddata.uncertainty.array[0:skyrow0, :] = 0.
self.action.args.ccddata.mask[objrow0:objrow1, :] = 1
self.action.args.ccddata.mask[0:skyrow0, :] = 1
self.action.args.ccddata.flags[objrow0:objrow1, :] = 64
self.action.args.ccddata.flags[0:skyrow0, :] = 64
sky[objrow0:objrow1, :] = 0.
sky[0:skyrow0, :] = 0.
obj[objrow0:objrow1, :] = 0.
obj[0:skyrow0, :] = 0.
cmnt = 'Aborted nod-and-shuffle observations'
objhdr['COMMENT'] = cmnt
skyhdr['COMMENT'] = cmnt
self.action.args.ccddata.header['COMMENT'] = cmnt
skyhdr['NASSCL'] = (skyscl, 'Scale factor applied to sky panel')
self.action.args.ccddata.header['NASSCL'] = (
skyscl, 'Scale factor applied to sky panel')
# log
self.logger.info("nod-and-shuffle subtracted, rows (sky0, 1, obj0,1): "
"%d, %d, %d, %d" %
(skyrow0, skyrow1, objrow0, objrow1))
# update headers
log_string = NandshuffSubtractSky.__module__
objhdr[key] = (False, keycom)
objhdr['HISTORY'] = log_string
skyhdr[key] = (False, keycom)
skyhdr['SKYOBS'] = (True, 'Sky observation?')
skyhdr['HISTORY'] = log_string
self.action.args.ccddata.header[key] = (True, keycom)
self.action.args.ccddata.header['HISTORY'] = log_string
# write out sky image
msname = ofn.split('.')[0] + '_' + target_type.lower() + '.fits'
out_sky = CCDData(sky, meta=skyhdr, unit=u_out)
kcwi_fits_writer(out_sky,
output_file=msname,
output_dir=self.config.instrument.output_directory)
# write out object image
obname = ofn.split('.')[0] + '_obj.fits'
out_obj = CCDData(obj, meta=objhdr, unit=u_out)
kcwi_fits_writer(out_obj,
output_file=obname,
output_dir=self.config.instrument.output_directory)
# update header keywords
self.action.args.ccddata.header['SKYMAST'] = (msname,
"Master sky filename")
# write out int image
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="intk")
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix="intk")
self.context.proctab.write_proctab()
self.logger.info(log_string)
return self.action.args
def _perform(self):
self.logger.info("Generating geometry maps")
log_string = GenerateMaps.__module__
if self.action.args.geometry_file is not None and \
os.path.exists(self.action.args.geometry_file):
with open(self.action.args.geometry_file, 'rb') as ifile:
geom = pickle.load(ifile)
# get geom params
xl0s = geom['xl0'] # lower slice pos limit
xl1s = geom['xl1'] # upper slice pos limit
invtf_list = geom['invtf']
wave0 = geom['wave0out']
dw = geom['dwout']
xsize = geom['xsize']
# Store original data
data_img = self.action.args.ccddata.data
ny = data_img.shape[0] # number of wavelength pixels
# Create map images
wave_map_img = np.full_like(data_img, fill_value=-1.)
xpos_map_img = np.full_like(data_img, fill_value=-1.)
slice_map_img = np.full_like(data_img, fill_value=-1.)
delta_map_img = np.full_like(data_img, fill_value=-1.)
# loop over slices
for isl in range(0, 24):
itrf = invtf_list[isl]
xl0 = xl0s[isl]
xl1 = xl1s[isl]
# loop over slice position
for ix in range(xl0, xl1):
coords = np.zeros((ny, 2))
# loop over wavelength pixels
for iy in range(0, ny):
coords[iy, 0] = ix - xl0
coords[iy, 1] = iy
ncoo = itrf(coords)
# loop over wavelength
for iy in range(0, ny):
if 0 <= ncoo[iy, 0] <= xsize:
slice_map_img[iy, ix] = isl
xpos_map_img[iy, ix] = ncoo[iy, 0]
wave_map_img[iy, ix] = ncoo[iy, 1] * dw + wave0
if iy > 0:
delta_map_img[iy, ix] = abs(
(ncoo[iy, 1] - ncoo[iy - 1, 1]) * dw)
# update header
self.action.args.ccddata.header['HISTORY'] = log_string
# Spatial geometry
self.action.args.ccddata.header['BARSEP'] = (
geom['barsep'], 'separation of bars (binned pix)')
self.action.args.ccddata.header['BAR0'] = (
geom['bar0'], 'first bar pixel position')
# Dichroic fraction
try:
dichroic_fraction = geom['dich_frac']
except AttributeError:
dichroic_fraction = 1.
self.action.args.ccddata.header['DICHFRAC'] = (dichroic_fraction,
'Dichroic Fraction')
# Wavelength ranges
self.action.args.ccddata.header['WAVALL0'] = (
geom['waveall0'], 'Low inclusive wavelength')
self.action.args.ccddata.header['WAVALL1'] = (
geom['waveall1'], 'High inclusive wavelength')
self.action.args.ccddata.header['WAVGOOD0'] = (
geom['wavegood0'], 'Low good wavelength')
self.action.args.ccddata.header['WAVGOOD1'] = (
geom['wavegood1'], 'High good wavelength')
self.action.args.ccddata.header['WAVMID'] = (geom['wavemid'],
'middle wavelength')
# Wavelength fit statistics
self.action.args.ccddata.header['AVWVSIG'] = (
geom['avwvsig'], 'Avg. bar wave sigma (Ang)')
self.action.args.ccddata.header['SDWVSIG'] = (
geom['sdwvsig'], 'Stdev. var wave sigma (Ang)')
# Pixel scales
self.action.args.ccddata.header['PXSCL'] = (
geom['pxscl'], 'Pixel scale along slice (deg)')
self.action.args.ccddata.header['SLSCL'] = (
geom['slscl'], 'Pixel scale perp. to slices (deg)')
# Geometry origins
self.action.args.ccddata.header['CBARSNO'] = (
geom['cbarsno'], 'Continuum bars image number')
self.action.args.ccddata.header['CBARSFL'] = (
geom['cbarsfl'], 'Continuum bars image filename')
self.action.args.ccddata.header['ARCNO'] = (geom['arcno'],
'Arc image number')
self.action.args.ccddata.header['ARCFL'] = (geom['arcfl'],
'Arc image filename')
self.action.args.ccddata.header['GEOMFL'] = (
self.action.args.geometry_file.split('/')[-1], 'Geometry file')
# output maps
header = self.action.args.ccddata.header
kcwi_fits_writer(
CCDData(wave_map_img, meta=header, unit=u.angstrom),
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="wavemap")
kcwi_fits_writer(
CCDData(xpos_map_img, meta=header, unit=u.pix),
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="posmap")
kcwi_fits_writer(
CCDData(slice_map_img, meta=header, unit=u.pix),
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="slicemap")
kcwi_fits_writer(
CCDData(delta_map_img, meta=header, unit=u.angstrom),
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="delmap")
else:
self.logger.error("Geom file not accessible")
self.logger.info(log_string)
return self.action.args
def _perform(self):
self.logger.info("Creating data cube")
log_string = MakeCube.__module__
# Are we interactive?
if self.config.instrument.plot_level >= 3:
do_inter = True
else:
do_inter = False
self.logger.info("Generating data cube")
# Find and read geometry transformation
tab = self.context.proctab.search_proctab(
frame=self.action.args.ccddata,
target_type='ARCLAMP',
nearest=True)
if not len(tab):
self.logger.error("No reference geometry, cannot make cube!")
self.action.args.ccddata.header['GEOMCOR'] = (
False, 'Geometry corrected?')
self.logger.info(log_string)
return self.action.args
self.logger.info("%d arc frames found" % len(tab))
ofn = strip_fname(tab['filename'][0]) + "_geom.pkl"
geom_file = os.path.join(self.config.instrument.cwd,
self.config.instrument.output_directory, ofn)
if os.path.exists(geom_file):
self.logger.info("Reading %s" % geom_file)
with open(geom_file, 'rb') as ifile:
geom = pickle.load(ifile)
# Slice size
xsize = geom['xsize']
ysize = geom['ysize']
out_cube = np.zeros((ysize, xsize, 24), dtype=np.float64)
out_vube = np.zeros((ysize, xsize, 24), dtype=np.float64)
out_mube = np.zeros((ysize, xsize, 24), dtype=np.uint8)
out_fube = np.zeros((ysize, xsize, 24), dtype=np.uint8)
out_oube = np.zeros((ysize, xsize, 24), dtype=np.float64)
out_sube = np.zeros((ysize, xsize, 24), dtype=np.float64)
out_dube = np.zeros((ysize, xsize, 24), dtype=np.float64)
# Store original data
data_img = self.action.args.ccddata.data
data_std = self.action.args.ccddata.uncertainty.array
data_msk = self.action.args.ccddata.mask
data_flg = self.action.args.ccddata.flags
# check for obj, sky images
obj = None
data_obj = None
sky = None
data_sky = None
if self.action.args.nasmask and self.action.args.numopen > 1:
ofn = self.action.args.name
objfn = strip_fname(ofn) + '_objf.fits'
full_path = os.path.join(
self.config.instrument.cwd,
self.config.instrument.output_directory, objfn)
if os.path.exists(full_path):
obj = kcwi_fits_reader(full_path)[0]
data_obj = obj.data
skyfn = strip_fname(ofn) + '_skyf.fits'
full_path = os.path.join(
self.config.instrument.cwd,
self.config.instrument.output_directory, skyfn)
if os.path.exists(full_path):
sky = kcwi_fits_reader(full_path)[0]
data_sky = sky.data
# check for geometry maps
dew = None
data_dew = None
if 'ARCLAMP' in self.action.args.imtype:
ofn = self.action.args.name
dewfn = strip_fname(ofn) + '_delmap.fits'
full_path = os.path.join(
self.config.instrument.cwd,
self.config.instrument.output_directory, dewfn)
if os.path.exists(full_path):
dew = kcwi_fits_reader(full_path)[0]
data_dew = dew.data
# Loop over 24 slices
my_arguments = []
for isl in range(0, 24):
arguments = {
'slice_number': isl,
'geom': geom,
'img': data_img,
'std': data_std,
'msk': data_msk,
'flg': data_flg,
'xsize': xsize,
'ysize': ysize,
'logger': self.logger
}
if obj is not None:
arguments['obj'] = data_obj
if sky is not None:
arguments['sky'] = data_sky
if dew is not None:
arguments['del'] = data_dew
my_arguments.append(arguments)
p = Pool()
results = p.map(make_cube_helper, list(my_arguments))
p.close()
self.logger.info("Building cube")
for partial_cube in results:
slice_number = partial_cube[0]
out_cube[:, :, slice_number] = partial_cube[1]
out_vube[:, :, slice_number] = partial_cube[2]
out_mube[:, :, slice_number] = partial_cube[3]
out_fube[:, :, slice_number] = partial_cube[4]
if obj is not None:
out_oube[:, :, slice_number] = partial_cube[5]
if sky is not None:
out_sube[:, :, slice_number] = partial_cube[6]
if dew is not None:
out_dube[:, :, slice_number] = partial_cube[7]
if self.config.instrument.plot_level >= 3:
for isl in range(0, 24):
warped = out_cube[:, :, isl]
ptitle = self.action.args.plotlabel + \
"WARPED Slice %d" % isl
p = figure(tooltips=[("x", "$x"), ("y", "$y"),
("value", "@image")],
title=ptitle,
x_axis_label="X (px)",
y_axis_label="Y (px)",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.x_range.range_padding = p.y_range.range_padding = 0
p.image([warped],
x=0,
y=0,
dw=xsize,
dh=ysize,
palette="Spectral11",
level="image")
bokeh_plot(p, self.context.bokeh_session)
if do_inter:
q = input("Next? <cr>, q to quit: ")
if 'Q' in q.upper():
do_inter = False
else:
time.sleep(self.config.instrument.plot_pause)
# Calculate some WCS parameters
# Get object pointing
try:
if self.action.args.nasmask:
rastr = self.action.args.ccddata.header['RABASE']
decstr = self.action.args.ccddata.header['DECBASE']
else:
rastr = self.action.args.ccddata.header['RA']
decstr = self.action.args.ccddata.header['DEC']
except KeyError:
try:
rastr = self.action.args.ccddata.header['TARGRA']
decstr = self.action.args.ccddata.header['TARGDEC']
except KeyError:
rastr = ''
decstr = ''
if len(rastr) > 0 and len(decstr) > 0:
try:
coord = SkyCoord(rastr, decstr, unit=(u.hourangle, u.deg))
except ValueError:
coord = None
else:
coord = None
# Get rotator position
if 'ROTPOSN' in self.action.args.ccddata.header:
rpos = self.action.args.ccddata.header['ROTPOSN']
else:
rpos = 0.
if 'ROTREFAN' in self.action.args.ccddata.header:
rref = self.action.args.ccddata.header['ROTREFAN']
else:
rref = 0.
skypa = rpos + rref
crota = math.radians(-(skypa + self.config.instrument.ROTOFF))
cdelt1 = -geom['slscl']
cdelt2 = geom['pxscl']
if coord is None:
ra = 0.
dec = 0.
crota = 1
else:
ra = coord.ra.degree
dec = coord.dec.degree
cd11 = cdelt1 * math.cos(crota)
cd12 = abs(cdelt2) * np.sign(cdelt1) * math.sin(crota)
cd21 = -abs(cdelt1) * np.sign(cdelt2) * math.sin(crota)
cd22 = cdelt2 * math.cos(crota)
crpix1 = 12.
crpix2 = xsize / 2.
crpix3 = 1.
porg = self.action.args.ccddata.header['PONAME']
ifunum = self.action.args.ifunum
if 'IFU' in porg:
if ifunum == 1:
off1 = 1.0
off2 = 4.0
elif ifunum == 2:
off1 = 1.0
off2 = 5.0
elif ifunum == 3:
off1 = 0.05
off2 = 5.6
else:
self.logger.warning("Unknown IFU number: %d" % ifunum)
off1 = 0.
off2 = 0.
off1 = off1 / float(self.action.args.xbinsize)
off2 = off2 / float(self.action.args.ybinsize)
crpix1 += off1
crpix2 += off2
# Update header
# Geometry corrected?
self.action.args.ccddata.header['GEOMCOR'] = (
True, 'Geometry corrected?')
#
# Spatial geometry
self.action.args.ccddata.header['BARSEP'] = (
geom['barsep'], 'separation of bars (binned pix)')
self.action.args.ccddata.header['BAR0'] = (
geom['bar0'], 'first bar pixel position')
# Wavelength ranges
if self.action.args.nasmask:
self.action.args.ccddata.header['WAVALL0'] = (
geom['wavensall0'], 'Low inclusive wavelength')
self.action.args.ccddata.header['WAVALL1'] = (
geom['wavensall1'], 'High inclusive wavelength')
self.action.args.ccddata.header['WAVGOOD0'] = (
geom['wavensgood0'], 'Low good wavelength')
self.action.args.ccddata.header['WAVGOOD1'] = (
geom['wavensgood1'], 'High good wavelength')
self.action.args.ccddata.header['WAVMID'] = (
geom['wavensmid'], 'middle wavelength')
else:
self.action.args.ccddata.header['WAVALL0'] = (
geom['waveall0'], 'Low inclusive wavelength')
self.action.args.ccddata.header['WAVALL1'] = (
geom['waveall1'], 'High inclusive wavelength')
self.action.args.ccddata.header['WAVGOOD0'] = (
geom['wavegood0'], 'Low good wavelength')
self.action.args.ccddata.header['WAVGOOD1'] = (
geom['wavegood1'], 'High good wavelength')
self.action.args.ccddata.header['WAVMID'] = (
geom['wavemid'], 'middle wavelength')
# Dichroic fraction
try:
dichroic_fraction = geom['dich_frac']
except AttributeError:
dichroic_fraction = 1.
self.action.args.ccddata.header['DICHFRAC'] = (dichroic_fraction,
'Dichroic Fraction')
# Wavelength fit statistics
self.action.args.ccddata.header['AVWVSIG'] = (
geom['avwvsig'], 'Avg. bar wave sigma (Ang)')
self.action.args.ccddata.header['SDWVSIG'] = (
geom['sdwvsig'], 'Stdev. var wave sigma (Ang)')
# Pixel scales
self.action.args.ccddata.header['PXSCL'] = (
geom['pxscl'], 'Pixel scale along slice (deg)')
self.action.args.ccddata.header['SLSCL'] = (
geom['slscl'], 'Pixel scale perp. to slices (deg)')
# Geometry origins
self.action.args.ccddata.header['CBARSNO'] = (
geom['cbarsno'], 'Continuum bars image number')
self.action.args.ccddata.header['CBARSFL'] = (
geom['cbarsfl'], 'Continuum bars image filename')
self.action.args.ccddata.header['ARCNO'] = (geom['arcno'],
'Arc image number')
self.action.args.ccddata.header['ARCFL'] = (geom['arcfl'],
'Arc image filename')
self.action.args.ccddata.header['GEOMFL'] = (
geom_file.split('/')[-1], 'Geometry file')
# WCS
self.action.args.ccddata.header['IFUPA'] = (
skypa, 'IFU position angle (degrees)')
self.action.args.ccddata.header['IFUROFF'] = (
self.config.instrument.ROTOFF, 'IFU-SKYPA offset (degrees)')
self.action.args.ccddata.header['WCSDIM'] = (
3, 'number of dimensions in WCS')
self.action.args.ccddata.header['WCSNAME'] = 'KCWI'
self.action.args.ccddata.header['EQUINOX'] = 2000.
self.action.args.ccddata.header['RADESYS'] = 'FK5'
self.action.args.ccddata.header['CTYPE1'] = 'RA---TAN'
self.action.args.ccddata.header['CTYPE2'] = 'DEC--TAN'
self.action.args.ccddata.header['CTYPE3'] = ('AWAV',
'Air Wavelengths')
self.action.args.ccddata.header['CUNIT1'] = ('deg', 'RA units')
self.action.args.ccddata.header['CUNIT2'] = ('deg', 'DEC units')
self.action.args.ccddata.header['CUNIT3'] = ('Angstrom',
'Wavelength units')
self.action.args.ccddata.header['CNAME1'] = ('KCWI RA', 'RA name')
self.action.args.ccddata.header['CNAME2'] = ('KCWI DEC',
'DEC name')
self.action.args.ccddata.header['CNAME3'] = ('KCWI Wavelength',
'Wavelength name')
self.action.args.ccddata.header['CRVAL1'] = (ra, 'RA zeropoint')
self.action.args.ccddata.header['CRVAL2'] = (dec, 'DEC zeropoint')
self.action.args.ccddata.header['CRVAL3'] = (
geom['wave0out'], 'Wavelength zeropoint')
self.action.args.ccddata.header['CRPIX1'] = (crpix1,
'RA reference pixel')
self.action.args.ccddata.header['CRPIX2'] = (crpix2,
'DEC reference pixel')
self.action.args.ccddata.header['CRPIX3'] = (
crpix3, 'Wavelength reference pixel')
self.action.args.ccddata.header['CD1_1'] = (
cd11, 'RA degrees per column pixel')
self.action.args.ccddata.header['CD2_1'] = (
cd21, 'DEC degrees per column pixel')
self.action.args.ccddata.header['CD1_2'] = (
cd12, 'RA degrees per row pixel')
self.action.args.ccddata.header['CD2_2'] = (
cd22, 'DEC degrees per row pixel')
self.action.args.ccddata.header['CD3_3'] = (
geom['dwout'], 'Wavelength Angstroms per pixel')
self.action.args.ccddata.header['LONPOLE'] = (
180.0, 'Native longitude of Celestial pole')
self.action.args.ccddata.header['LATPOLE'] = (
0.0, 'Native latitude of Celestial pole')
# write out cube
self.action.args.ccddata.header['HISTORY'] = log_string
self.action.args.ccddata.data = out_cube
self.action.args.ccddata.uncertainty.array = out_vube
self.action.args.ccddata.mask = out_mube
self.action.args.ccddata.flags = out_fube
# write out int image
kcwi_fits_writer(
self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="icube")
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix="icube",
filename=self.action.args.name)
self.context.proctab.write_proctab()
# check for obj, sky outputs
if obj is not None:
out_obj = CCDData(out_oube,
meta=self.action.args.ccddata.header,
unit=self.action.args.ccddata.unit)
kcwi_fits_writer(
out_obj,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="ocube")
if sky is not None:
out_sky = CCDData(out_sube,
meta=self.action.args.ccddata.header,
unit=self.action.args.ccddata.unit)
kcwi_fits_writer(
out_sky,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="scube")
# check for dew outputs
if dew is not None:
out_dew = CCDData(out_dube,
meta=self.action.args.ccddata.header,
unit=dew.unit)
kcwi_fits_writer(
out_dew,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="dcube")
else:
self.logger.error("Geometry file not found: %s" % geom_file)
self.logger.info(log_string)
return self.action.args
def _perform(self):
self.logger.info("Tracing continuum bars")
if self.config.instrument.plot_level >= 1:
do_plot = True
else:
do_plot = False
if len(self.action.args.middle_centers) < 1:
self.logger.error("No bars found")
elif not self.action.args.bar_avg:
self.logger.error("No threshold for tracing")
else:
# initialize
samp = int(80 / self.action.args.ybinsize)
win = self.action.args.window
bar_thresh = self.action.args.bar_avg
self.logger.info("Tracing bars with threshold of %.1f" % bar_thresh)
xi = [] # x input
xo = [] # x output
yi = [] # y input (and output)
barid = [] # bar id number
slid = [] # slice id number
# loop over bars
for barn, barx in enumerate(self.action.args.middle_centers):
# nearest pixel to bar center
barxi = int(barx + 0.5)
self.logger.info("bar number %d is at %.3f" % (barn, barx))
# middle row data
xi.append(barx)
xo.append(barx)
yi.append(self.action.args.middle_row)
barid.append(barn)
slid.append(int(barn/5))
# trace up
samy = self.action.args.middle_row + samp
done = False
while samy < (self.action.args.ccddata.data.shape[0] - win) \
and not done:
ys = np.median(
self.action.args.ccddata.data[(samy - win):
(samy + win + 1),
(barxi - win):
(barxi + win + 1)],
axis=0)
ys = ys - np.nanmin(ys)
if np.nanmax(ys) > bar_thresh and np.nansum(ys) > 0:
xs = list(range(barxi - win, barxi + win + 1))
xc = np.nansum(xs * ys) / np.nansum(ys)
xi.append(xc)
xo.append(barx)
yi.append(samy)
barid.append(barn)
slid.append(int(barn/5))
barxi = int(xc)
else:
done = True
samy += samp
# trace down
# nearest pixel to bar center
barxi = int(barx + 0.5)
samy = self.action.args.middle_row - samp
done = False
while samy >= win and not done:
ys = np.median(
self.action.args.ccddata.data[(samy - win):
(samy + win + 1),
(barxi - win):
(barxi + win + 1)],
axis=0)
ys = ys - np.nanmin(ys)
if np.nanmax(ys) > bar_thresh and np.nansum(ys) > 0:
xs = list(range(barxi - win, barxi + win + 1))
xc = np.sum(xs * ys) / np.sum(ys)
xi.append(xc)
xo.append(barx)
yi.append(samy)
barid.append(barn)
slid.append(int(barn / 5))
barxi = int(xc)
else:
done = True
samy -= samp
# end loop over bars
# create source and destination coords
yo = yi
dst = np.column_stack((xi, yi))
src = np.column_stack((xo, yo))
if do_plot:
# output filename stub
trcfnam = "bars_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
# plot them
p = figure(title=self.action.args.plotlabel +
'SPATIAL CONTROL POINTS',
x_axis_label="CCD X (px)", y_axis_label="CCD Y (px)",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.scatter(xi, yi, marker='x', size=2, color='blue')
p.scatter(self.action.args.middle_centers,
[self.action.args.middle_row]*120, color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=trcfnam+".png")
trace = {
'src': src,
'dst': dst,
'barid': barid,
'slid': slid,
'MIDROW': self.action.args.middle_row,
'WINDOW': self.action.args.window,
'REFDELX': self.action.args.reference_delta_x,
'CBARSNO': self.action.args.contbar_image_number,
'CBARSFL': self.action.args.contbar_image}
# in this line we pass the trace information to an argument
# instead of writing it to a table
self.context.trace = trace
ofname = strip_fname(self.action.args.contbar_image) + \
"_trace.fits"
write_table(table=[src, dst, barid, slid],
names=('src', 'dst', 'barid', 'slid'),
output_dir=os.path.join(
self.config.instrument.cwd,
self.config.instrument.output_directory),
output_name=ofname,
clobber=self.config.instrument.clobber,
comment=['Source and destination fiducial points',
'Derived from KCWI continuum bars images',
'For defining spatial transformation'],
keywords={'MIDROW': (self.action.args.middle_row,
"Middle Row of image"),
'WINDOW': (self.action.args.window,
"Window for bar"),
'REFDELX': (
self.action.args.reference_delta_x,
"Reference bar sep in px"),
'CBARSNO': (
self.action.args.contbar_image_number,
"Cont. bars image number"),
'CBARSFL': (self.action.args.contbar_image,
"Cont. bars image")})
if self.config.instrument.saveintims:
from kcwidrp.primitives.kcwi_file_primitives import \
kcwi_fits_writer
from skimage import transform as tf
# fit transform
self.logger.info("Fitting spatial control points")
tform = tf.estimate_transform('polynomial', src, dst, order=3)
self.logger.info("Transforming bars image")
warped = tf.warp(self.action.args.ccddata.data, tform)
# write out warped image
self.action.args.ccddata.data = warped
kcwi_fits_writer(
self.action.args.ccddata, output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix='warped')
self.logger.info("Transformed bars produced")
log_string = TraceBars.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
return self.action.args
def _perform(self):
do_plot = (self.config.instrument.plot_level >= 3)
self.logger.info("Extracting arc spectra")
# Double check
if not self.action.args.original_filename:
self.logger.error("No traces found")
return self.action.args
# All is ready
original_filename = self.action.args.original_filename
self.logger.info("Trace table found: %s" % original_filename)
# trace = read_table(tab=tab, indir='redux', suffix='trace')
# Find and read control points from continuum bars
if hasattr(self.context, 'trace'):
trace = self.context.trace
else:
trace = read_table(
input_dir=os.path.join(self.config.instrument.cwd,
self.config.instrument.output_directory),
file_name=original_filename)
self.context.trace = {}
for key in trace.meta.keys():
self.context.trace[key] = trace.meta[key]
middle_row = self.context.trace['MIDROW']
window = self.context.trace['WINDOW']
self.action.args.reference_bar_separation = self.context.trace[
'REFDELX']
self.action.args.contbar_image_number = self.context.trace['CBARSNO']
self.action.args.contbar_image = self.context.trace['CBARSFL']
self.action.args.arc_number = self.action.args.ccddata.header['FRAMENO']
self.action.args.arc_image = self.action.args.ccddata.header['OFNAME']
self.action.args.source_control_points = trace['src']
self.action.args.destination_control_points = trace['dst']
self.action.args.bar_id = trace['barid']
self.action.args.slice_id = trace['slid']
self.logger.info("Fitting spatial control points")
transformation = tf.estimate_transform(
'polynomial', self.action.args.source_control_points,
self.action.args.destination_control_points, order=3)
self.logger.info("Transforming arc image")
warped_image = tf.warp(self.action.args.ccddata.data, transformation)
# Write warped arcs if requested
if self.config.instrument.saveintims:
from kcwidrp.primitives.kcwi_file_primitives import kcwi_fits_writer
# write out warped image
self.action.args.ccddata.data = warped_image
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="warped")
self.logger.info("Transformed arcs produced")
# extract arcs
self.logger.info("Extracting arcs")
arcs = []
# sectors for bkgnd subtraction
sectors = 16
for xyi, xy in enumerate(self.action.args.source_control_points):
if xy[1] == middle_row:
xi = int(xy[0]+0.5)
arc = np.median(
warped_image[:, (xi - window):(xi + window + 1)], axis=1)
# divide spectrum into sectors
div = int((len(arc)-100) / sectors)
# get minimum for each sector
xv = []
yv = []
for i in range(sectors):
mi = np.nanargmin(arc[50+i*div:50+(i+1)*div])
mn = np.nanmin(arc[50+i*div:50+(i+1)*div])
xv.append(mi+50+i*div)
yv.append(mn)
# fit minima to model background
res = np.polyfit(xv, yv, 3)
xp = np.arange(len(arc))
bkg = np.polyval(res, xp) # resulting model
# plot if requested
if do_plot:
p = figure(title=self.action.args.plotlabel + "ARC # %d" %
len(arcs),
x_axis_label="Y CCD Pixel",
y_axis_label="Flux",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(xp, arc, legend_label='Arc', color='blue')
p.line(xp, bkg, legend_label='Bkg', color='red')
bokeh_plot(p, self.context.bokeh_session)
q = input("Next? <cr>, q to quit: ")
if 'Q' in q.upper():
do_plot = False
# subtract model background
arc -= bkg
# add to arcs list
arcs.append(arc)
# Did we get the correct number of arcs?
if len(arcs) == self.config.instrument.NBARS:
self.logger.info("Extracted %d arcs" % len(arcs))
self.context.arcs = arcs
else:
self.logger.error("Did not extract %d arcs, extracted %d" %
(self.config.instrument.NBARS, len(arcs)))
log_string = ExtractArcs.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
return self.action.args
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation
"""
self.logger.info("Creating master sky")
suffix = self.action.args.new_type.lower()
# get root for maps
tab = self.context.proctab.n_proctab(
frame=self.action.args.ccddata, target_type='ARCLAMP',
target_group=self.action.args.groupid)
if len(tab) <= 0:
self.logger.error("Geometry not solved!")
return self.action.args
groot = tab['OFNAME'][0].split('.fits')[0]
# Wavelength map image
wmf = groot + '_wavemap.fits'
self.logger.info("Reading image: %s" % wmf)
wavemap = kcwi_fits_reader(
os.path.join(os.path.dirname(self.action.args.name), 'redux',
wmf))[0]
# Slice map image
slf = groot + '_slicemap.fits'
self.logger.info("Reading image: %s" % slf)
slicemap = kcwi_fits_reader(
os.path.join(os.path.dirname(self.action.args.name), 'redux',
slf))[0]
# Position map image
pof = groot + '_posmap.fits'
self.logger.info("Reading image: %s" % pof)
posmap = kcwi_fits_reader(
os.path.join(os.path.dirname(self.action.args.name), 'redux',
pof))[0]
posmax = np.nanmax(posmap.data)
posbuf = int(10. / self.action.args.xbinsize)
# wavelength region
wavegood0 = wavemap.header['WAVGOOD0']
wavegood1 = wavemap.header['WAVGOOD1']
waveall0 = wavemap.header['WAVALL0']
waveall1 = wavemap.header['WAVALL1']
# get image size
sm_sz = self.action.args.ccddata.data.shape
# sky masking
# default is no masking (True = mask, False = don't mask)
binary_mask = np.zeros(sm_sz, dtype=bool)
# was sky masking requested?
if self.action.args.skymask:
if os.path.exists(self.action.args.skymask):
self.logger.info("Reading sky mask file: %s"
% self.action.args.skymask)
hdul = fits.open(self.action.args.skymask)
binary_mask = hdul[0].data
# verify size match
bm_sz = binary_mask.shape
if bm_sz[0] != sm_sz[0] or bm_sz[1] != sm_sz[1]:
self.logger.warning("Sky mask size mis-match: "
"masking disabled")
binary_mask = np.zeros(sm_sz, dtype=bool)
else:
self.logger.warning("Sky mask image not found: %s"
% self.action.args.skymask)
# count masked pixels
tmsk = len(np.nonzero(np.where(binary_mask.flat, True, False))[0])
self.logger.info("Number of pixels masked = %d" % tmsk)
finiteflux = np.isfinite(self.action.args.ccddata.data.flat)
# get un-masked points mapped to exposed regions on CCD
# handle dichroic bad region
if self.action.args.dich:
if self.action.args.camera == 0: # Blue
q = [i for i, v in enumerate(slicemap.data.flat)
if 0 <= v <= 23 and
posbuf < posmap.data.flat[i] < (posmax - posbuf) and
waveall0 <= wavemap.data.flat[i] <= waveall1 and
not (v > 20 and wavemap.data.flat[i] > 5600.) and
finiteflux[i] and not binary_mask.flat[i]]
else: # Red
q = [i for i, v in enumerate(slicemap.data.flat)
if 0 <= v <= 23 and
posbuf < posmap.data.flat[i] < (posmax - posbuf) and
waveall0 <= wavemap.data.flat[i] <= waveall1 and
not (v > 20 and wavemap.data.flat[i] < 5600.) and
finiteflux[i] and not binary_mask.flat[i]]
else:
q = [i for i, v in enumerate(slicemap.data.flat)
if 0 <= v <= 23 and
posbuf < posmap.data.flat[i] < (posmax - posbuf) and
waveall0 <= wavemap.data.flat[i] <= waveall1 and
finiteflux[i] and not binary_mask.flat[i]]
# get all points mapped to exposed regions on the CCD (for output)
qo = [i for i, v in enumerate(slicemap.data.flat)
if 0 <= v <= 23 and posmap.data.flat[i] >= 0 and
waveall0 <= wavemap.data.flat[i] <= waveall1 and
finiteflux[i]]
# extract relevant image values
fluxes = self.action.args.ccddata.data.flat[q]
# relevant wavelengths
waves = wavemap.data.flat[q]
self.logger.info("Number of fit waves = %d" % len(waves))
# keep output wavelengths
owaves = wavemap.data.flat[qo]
self.logger.info("Number of output waves = %d" % len(owaves))
# sort on wavelength
s = np.argsort(waves)
waves = waves[s]
fluxes = fluxes[s]
# knots per pixel
knotspp = self.config.instrument.KNOTSPP
n = int(sm_sz[0] * knotspp)
# calculate break points for b splines
bkpt = np.min(waves) + np.arange(n+1) * \
(np.max(waves) - np.min(waves)) / n
# log
self.logger.info("Nknots = %d, min = %.2f, max = %.2f (A)" %
(n, np.min(bkpt), np.max(bkpt)))
# do bspline fit
sft0, gmask = Bspline.iterfit(waves, fluxes, fullbkpt=bkpt,
upper=1, lower=1)
gp = [i for i, v in enumerate(gmask) if v]
yfit1, _ = sft0.value(waves)
self.logger.info("Number of good points = %d" % len(gp))
# check result
if np.max(yfit1) < 0:
self.logger.warning("B-spline failure")
if n > 2000:
if n == 5000:
n = 2000
if n == 8000:
n = 5000
# calculate breakpoints
bkpt = np.min(waves) + np.arange(n + 1) * \
(np.max(waves) - np.min(waves)) / n
# log
self.logger.info("Nknots = %d, min = %.2f, max = %.2f (A)" %
(n, np.min(bkpt), np.max(bkpt)))
# do bspline fit
sft0, gmask = Bspline.iterfit(waves, fluxes, fullbkpt=bkpt,
upper=1, lower=1)
yfit1, _ = sft0.value(waves)
if np.max(yfit1) <= 0:
self.logger.warning("B-spline final failure, sky is zero")
# get values at original wavelengths
yfit, _ = sft0.value(owaves)
# for plotting
gwaves = waves[gp]
gfluxes = fluxes[gp]
npts = len(gwaves)
stride = int(npts / 8000.)
xplt = gwaves[::stride]
yplt = gfluxes[::stride]
fplt, _ = sft0.value(xplt)
yrng = [np.min(yplt), np.max(yplt)]
self.logger.info("Stride = %d" % stride)
# plot, if requested
if self.config.instrument.plot_level >= 1:
# output filename stub
skyfnam = "sky_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
p = figure(
title=self.action.args.plotlabel + ' Master Sky',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xplt, yplt, size=1, line_alpha=0., fill_color='purple',
legend_label='Data')
p.line(xplt, fplt, line_color='red', legend_label='Fit')
p.line([wavegood0, wavegood0], yrng, line_color='green')
p.line([wavegood1, wavegood1], yrng, line_color='green')
p.y_range.start = yrng[0]
p.y_range.end = yrng[1]
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=skyfnam+".png")
# create sky image
sky = np.zeros(self.action.args.ccddata.data.shape, dtype=float)
sky.flat[qo] = yfit
# store original data, header
img = self.action.args.ccddata.data
hdr = self.action.args.ccddata.header.copy()
self.action.args.ccddata.data = sky
# get master sky output name
ofn = self.action.args.ccddata.header['OFNAME']
msname = ofn.split('.fits')[0] + '_' + suffix + '.fits'
log_string = MakeMasterSky.__module__
self.action.args.ccddata.header['IMTYPE'] = self.action.args.new_type
self.action.args.ccddata.header['HISTORY'] = log_string
self.action.args.ccddata.header['SKYMODEL'] = (True, 'sky model image?')
self.action.args.ccddata.header['SKYIMAGE'] = \
(ofn, 'image used for sky model')
if tmsk > 0:
self.action.args.ccddata.header['SKYMSK'] = (True,
'was sky masked?')
# self.action.args.ccddata.header['SKYMSKF'] = (skymf,
# 'sky mask file')
else:
self.action.args.ccddata.header['SKYMSK'] = (False,
'was sky masked?')
self.action.args.ccddata.header['WAVMAPF'] = wmf
self.action.args.ccddata.header['SLIMAPF'] = slf
self.action.args.ccddata.header['POSMAPF'] = pof
# output master sky
kcwi_fits_writer(self.action.args.ccddata, output_file=msname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix=suffix,
newtype=self.action.args.new_type)
self.context.proctab.write_proctab()
# restore original image
self.action.args.ccddata.data = img
self.action.args.ccddata.header = hdr
self.logger.info(log_string)
return self.action.args
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation
"""
method = 'average'
suffix = self.action.args.new_type.lower()
combine_list = list(self.combine_list['filename'])
# get master bias output name
# mbname = combine_list[-1].split('.fits')[0] + '_' + suffix + '.fits'
mbname = master_bias_name(self.action.args.ccddata)
stack = []
stackf = []
for bias in combine_list:
stackf.append(bias)
# using [0] drops the table
stack.append(kcwi_fits_reader(bias)[0])
stacked = ccdproc.combine(stack,
method=method,
sigma_clip=True,
sigma_clip_low_thresh=None,
sigma_clip_high_thresh=2.0)
stacked.header['IMTYPE'] = self.action.args.new_type
stacked.header['NSTACK'] = (len(combine_list),
'number of images stacked')
stacked.header['STCKMETH'] = (method, 'method used for stacking')
for ii, fname in enumerate(stackf):
fname_base = os.path.basename(fname)
stacked.header['STACKF%d' % (ii + 1)] = (fname_base,
"stack input file")
# for readnoise stats use 2nd and 3rd bias
diff = stack[1].data.astype(np.float32) - \
stack[2].data.astype(np.float32)
namps = stack[1].header['NVIDINP']
for ia in range(namps):
# get gain
gain = stacked.header['GAIN%d' % (ia + 1)]
# get amp section
sec, rfor = parse_imsec(stacked.header['DSEC%d' % (ia + 1)])
noise = diff[sec[0]:(sec[1] + 1), sec[2]:(sec[3] + 1)]
noise = np.reshape(noise, noise.shape[0]*noise.shape[1]) * \
gain / 1.414
# get stats on noise
c, low, upp = sigmaclip(noise, low=3.5, high=3.5)
bias_rn = c.std()
self.logger.info("Amp%d read noise from bias in e-: %.3f" %
((ia + 1), bias_rn))
stacked.header['BIASRN%d' % (ia + 1)] = \
(float("%.3f" % bias_rn), "RN in e- from bias")
if self.config.instrument.plot_level >= 1:
# output filename stub
biasfnam = "bias_%05d_amp%d_rdnoise" % \
(self.action.args.ccddata.header['FRAMENO'], ia+1)
plabel = '[ Img # %d' % self.action.args.ccddata.header[
'FRAMENO']
plabel += ' (Bias)'
plabel += ' %s' % self.action.args.ccddata.header['BINNING']
plabel += ' %s' % self.action.args.ccddata.header['AMPMODE']
plabel += ' %d' % self.action.args.ccddata.header['GAINMUL']
plabel += ' %s' % ('fast' if self.action.args.ccddata.
header['CCDMODE'] else 'slow')
plabel += ' ] '
hist, edges = np.histogram(noise,
range=(low, upp),
density=False,
bins=50)
x = np.linspace(low, upp, 500)
pdf = np.max(hist) * np.exp(-x**2 / (2. * bias_rn**2))
p = figure(title=plabel + 'BIAS NOISE amp %d = %.3f' %
(ia + 1, bias_rn),
x_axis_label='e-',
y_axis_label='N',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.quad(top=hist,
bottom=0,
left=edges[:-1],
right=edges[1:],
fill_color="navy",
line_color="white",
alpha=0.5)
p.line(x,
pdf,
line_color="#ff8888",
line_width=4,
alpha=0.7,
legend_label="PDF")
p.line([-bias_rn, -bias_rn], [0, np.max(hist)],
color='red',
legend_label="Sigma")
p.line([bias_rn, bias_rn], [0, np.max(hist)], color='red')
p.y_range.start = 0
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=biasfnam + ".png")
log_string = MakeMasterBias.__module__
stacked.header['HISTORY'] = log_string
self.logger.info(log_string)
kcwi_fits_writer(stacked,
output_file=mbname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=stacked,
suffix=suffix,
newtype=self.action.args.new_type,
filename=self.action.args.name)
self.context.proctab.write_proctab()
return Arguments(name=mbname)
def _perform(self):
self.logger.info("Making inverse sensitivity curve")
suffix = 'invsens'
stdname = self.action.args.stdname
# get size
sz = self.action.args.ccddata.data.shape
# default pixel ranges
z = np.arange(sz[0])
z0 = 175
z1 = sz[0] - 175
# get exposure time
expt = self.action.args.ccddata.header['XPOSURE']
if expt == 0.:
self.logger.warning("No exposure time found, setting to 1s")
expt = 1.
else:
self.logger.info("Using exposure time of %.1f" % expt)
# get wavelength scale
w0 = self.action.args.ccddata.header['CRVAL3']
dw = self.action.args.ccddata.header['CD3_3']
crpixw = self.action.args.ccddata.header['CRPIX3']
# get all good wavelength range
wgoo0 = self.action.args.ccddata.header['WAVGOOD0']
if wgoo0 < 3650:
wgoo0 = 3650.
wgoo1 = self.action.args.ccddata.header['WAVGOOD1']
# get all inclusive wavelength range
wall0 = self.action.args.ccddata.header['WAVALL0']
wall1 = self.action.args.ccddata.header['WAVALL1']
# get DAR padding in y
pad_y = self.action.args.ccddata.header['DARPADY']
# get sky subtraction status
skycor = self.action.args.ccddata.header['SKYCOR']
# get telescope and atm. correction
if 'TELESCOP' in self.action.args.ccddata.header:
tel = self.action.args.ccddata.header['TELESCOP']
else:
tel = 'KeckI'
if 'Keck' in tel:
area = 760000.0
else:
area = -1.0
# compute good y pixel ranges
if w0 > 0. and dw > 0. and wgoo0 > 0. and wgoo1 > 0.:
z0 = int((wgoo0 - w0) / dw) + 10
z1 = int((wgoo1 - w0) / dw) - 10
# wavelength scale
w = w0 + z * dw
# good spatial range
gy0 = pad_y if pad_y > 1 else 1
gy1 = sz[1] - (pad_y if pad_y > 2 else 2)
# log results
self.logger.info("Invsen. Pars: Y0, Y1, Z0, Z1, Wav0, Wav1: "
"%d, %d, %d, %d, %.3f, %.3f" %
(gy0, gy1, z0, z1, w[z0], w[z1]))
# central wavelength
cwv = self.action.args.cwave
# find standard in slices
# sum over wavelength
tot = np.sum(self.action.args.ccddata.data[z0:z1, gy0:gy1, :], 0)
# yy = np.arange(gy1-gy0) + gy0
mxsl = -1.
mxsg = 0.
# for each slice
for i in range(sz[2]):
tstd = float(np.nanstd(tot[:, i]))
if tstd > mxsg:
mxsg = tstd
mxsl = i
# relevant slices
sl0 = (mxsl - 3) if mxsl >= 3 else 0
sl1 = (mxsl + 3) if (mxsl + 3) <= sz[2] - 1 else sz[2] - 1
# get y position of std
cy, _ = find_peaks(tot[:, mxsl], height=np.nanmean(tot[:, mxsl]))
cy = int(cy[0]) + gy0
# log results
self.logger.info("Std lices: max, sl0, sl1, spatial cntrd: "
"%d, %d, %d, %.2f" % (mxsl, sl0, sl1, cy))
# get dwave spectrum
ofn = self.action.args.ccddata.header['OFNAME']
delfn = ofn.split('.')[0] + '_dcubed.fits'
full_path = os.path.join(os.path.dirname(self.action.args.name),
self.config.instrument.output_directory,
delfn)
if os.path.exists(full_path):
dew = kcwi_fits_reader(full_path)[0]
dwspec = dew.data[:, cy, mxsl]
zeros = np.where(dwspec == 0)
if len(zeros) > 0:
dwspec[zeros] = dw
else:
dwspec = np.zeros(sz[0]) + dw
# copy of input cube
scub = self.action.args.ccddata.data.copy()
# sky window width in pixels
skywin = int(self.config.instrument.psfwid / self.action.args.xbinsize)
# do sky subtraction, if needed
if not skycor:
self.logger.warning("Sky should have been subtraced already")
# apply extinction correction
ucub = scub.copy() # uncorrected cube
kcwi_correct_extin(scub,
self.action.args.ccddata.header,
logger=self.logger)
# get slice spectra y limits
sy0 = (cy - skywin) if (cy - skywin) > 0 else 0
sy1 = (cy + skywin) if (cy + skywin) < (sz[1] - 1) else (sz[1] - 1)
# sum over y range
slspec = np.sum(scub[:, sy0:sy1, :], 1)
ulspec = np.sum(ucub[:, sy0:sy1, :], 1)
# sum over slices
obsspec = np.sum(slspec[:, sl0:sl1], 1)
ubsspec = np.sum(ulspec[:, sl0:sl1], 1)
# convert to e-/second
obsspec /= expt
ubsspec /= expt
# check for zeros
zeros = np.where(obsspec == 0.)
if len(zeros) > 0:
obsmean = np.nanmean(obsspec)
obsspec[zeros] = obsmean
# read in standard star spectrum
hdul = pf.open(self.action.args.stdfile)
swl = hdul[1].data['WAVELENGTH']
sflx = hdul[1].data['FLUX']
sfw = hdul[1].data['FWHM']
hdul.close()
# get region of interest
sroi = [i for i, v in enumerate(swl) if w[0] <= v <= w[-1]]
nsroi = len(sroi)
if nsroi <= 0:
self.logger.error("No standard wavelengths in common")
return self.action.args
# expand range after checking for edges
if sroi[0] > 0:
sroi.insert(0, sroi[0] - 1)
nsroi += 1
if sroi[-1] < (len(swl) - 1):
sroi.append(sroi[-1] + 1)
nsroi += 1
# very sparsely sampled w.r.t. object
if nsroi <= 1:
self.logger.error("Not enough standard points")
return self.action.args
self.logger.info("Number of standard point = %d" % nsroi)
# how many points?
# do_line = nsroi > 20
swl = swl[sroi]
sflx = sflx[sroi]
sfw = sfw[sroi]
fwhm = np.max(sfw)
self.logger.info("Reference spectrum FWHM used = %.1f (A)" % fwhm)
# resample standard onto our wavelength grid
rsint = interp1d(swl, sflx, kind='cubic', fill_value='extrapolate')
rsflx = rsint(w)
# get effective inverse sensitivity
invsen = rsflx / obsspec
# convert to photons/s/cm^2/(wl bin = dw)
rspho = 5.03411250e7 * rsflx * w * dw
# get effective area
earea = ubsspec / rspho
# correct to native bins
earea *= dw / dwspec
# Balmer lines
blines = [6563., 4861., 4341., 4102., 3970., 3889., 3835.]
# default values (for BM)
bwid = 0.008 # fractional width to mask
ford = 9 # fit order
if 'BL' in self.action.args.grating:
bwid = 0.004
ford = 7
elif 'BH' in self.action.args.grating:
bwid = 0.012
ford = 9
# Adjust for dichroic fraction
try:
dichroic_fraction = self.action.args.ccddata.header['DICHFRAC']
except KeyError:
dichroic_fraction = 1.
ford = int(ford * dichroic_fraction)
if ford < 3:
ford = 3
self.logger.info("Fitting Invsens with polynomial order %d" % ford)
bwids = [bl * bwid for bl in blines]
# fit inverse sensitivity and effective area
# get initial region of interest
wl_good = [i for i, v in enumerate(w) if wgoo0 <= v <= wgoo1]
nwl_good = len(wl_good)
if nwl_good <= 0:
self.logger.error("No good wavelengths to fit")
return self.action.args
wlm0 = wgoo0
wlm1 = wgoo1
# interactively set wavelength limits
if self.config.instrument.plot_level >= 1:
yran = [np.min(obsspec), np.max(obsspec)]
source = ColumnDataSource(data=dict(x=w, y=obsspec))
done = False
while not done:
p = figure(tooltips=[("x", "@x{0,0.0}"), ("y", "@y{0,0.0}")],
title=self.action.args.plotlabel + ' Obs Spec',
x_axis_label='Wave (A)',
y_axis_label='Intensity (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line('x', 'y', line_color='black', source=source)
p.line([wgoo0, wgoo0],
yran,
line_color='green',
legend_label='WAVGOOD')
p.line([wgoo1, wgoo1], yran, line_color='green')
p.line([wlm0, wlm0],
yran,
line_color='blue',
legend_label='LIMITS')
p.line([wlm1, wlm1], yran, line_color='blue')
p.line([cwv, cwv], yran, line_color='red', legend_label='CWAV')
set_plot_lims(p, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(p, self.context.bokeh_session)
print("WL limits: %.1f - %.1f" % (wlm0, wlm1))
qstr = input("New? <float> <float>, <cr> - done: ")
if len(qstr) <= 0:
done = True
else:
try:
wlm0 = float(qstr.split()[0])
wlm1 = float(qstr.split()[1])
if wlm1 < wlm0 or wlm0 < wall0 or wlm1 > wall1:
wlm0 = wgoo0
wlm1 = wgoo1
print("range/order error, try again")
except (IndexError, ValueError):
wlm0 = wgoo0
wlm1 = wgoo1
print("format error, try again")
# update region of interest
wl_good = [i for i, v in enumerate(w) if wlm0 <= v <= wlm1]
nwl_good = len(wl_good)
# END: interactively set wavelength limits
# Now interactively identify lines
if self.config.instrument.plot_level >= 1:
yran = [np.min(obsspec), np.max(obsspec)]
# source = ColumnDataSource(data=dict(x=w, y=obsspec))
done = False
while not done:
p = figure(tooltips=[("x", "@x{0.0}"), ("y", "@y{0.0}")],
title=self.action.args.plotlabel + ' Obs Spec',
x_axis_label='Wave (A)',
y_axis_label='Intensity (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(w, obsspec, line_color='black')
p.line([wgoo0, wgoo0],
yran,
line_color='green',
legend_label='WAVGOOD')
p.line([wgoo1, wgoo1], yran, line_color='green')
p.line([wlm0, wlm0],
yran,
line_color='blue',
legend_label='LIMITS')
p.line([wlm1, wlm1], yran, line_color='blue')
p.line([cwv, cwv], yran, line_color='red', legend_label='CWAV')
for il, bl in enumerate(blines):
if wall0 < bl < wall1:
p.line([bl, bl], yran, line_color='orange')
p.line([bl - bwids[il], bl - bwids[il]],
yran,
line_color='orange',
line_dash='dashed')
p.line([bl + bwids[il], bl + bwids[il]],
yran,
line_color='orange',
line_dash='dashed')
set_plot_lims(p, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(p, self.context.bokeh_session)
qstr = input("New lines? <float> [<float>] ... (A), "
"<cr> - done: ")
if len(qstr) <= 0:
done = True
else:
for lstr in qstr.split():
try:
new_line = float(lstr)
except ValueError:
print("bad line: %s" % lstr)
continue
if wlm0 < new_line < wlm1:
blines.append(new_line)
bwids.append(bwid * new_line)
else:
print("line outside range: %s" % lstr)
# END: interactively identify lines
# set up fitting vectors, flux, waves, measure errors
sf = invsen[wl_good] # dependent variable
af = earea[wl_good] # effective area
wf = w[wl_good] # independent variable
bf = obsspec[wl_good] # input electrons
rf = rsflx[wl_good] # reference flux
mw = np.ones(nwl_good, dtype=float) # weights
use = np.ones(nwl_good, dtype=int) # toggles for usage
# loop over Balmer lines
for il, bl in enumerate(blines):
roi = [
i for i, v in enumerate(wf)
if (bl - bwids[il]) <= v <= (bl + bwids[il])
]
nroi = len(roi)
if nroi > 0:
use[roi] = 0
self.logger.info("Masking line at %.1f +- %.4f (A)" %
(bl, bwids[il]))
# ignore bad points by setting large errors
mf = []
ef = []
ww = []
used = []
not_used = []
for i in range(len(use)):
if use[i] == 1:
used.append(i)
else:
mw[i] = 1.e-9
mf.append(sf[i])
ef.append(100. * af[i] / area)
ww.append(wf[i])
not_used.append(i)
# initial polynomial fit of inverse sensitivity
wf0 = np.min(wf)
res = np.polyfit(wf - wf0, sf, deg=ford, w=mw)
finvsen = np.polyval(res, w - wf0)
sinvsen = np.polyval(res, wf - wf0)
calspec = obsspec * finvsen
scalspec = bf * sinvsen
# initial polynomial fit of effective area
res = np.polyfit(wf - wf0, af, ford, w=mw)
fearea = np.polyval(res, w - wf0)
# calculate residuals
resid = 100.0 * (scalspec - rf) / rf
if len(not_used) > 0:
rbad = resid[not_used]
else:
rbad = None
rsd_mean = float(np.nanmean(resid[used]))
rsd_stdv = float(np.nanstd(resid[used]))
self.logger.info("Calibration residuals = %f +- %f %%" %
(rsd_mean, rsd_stdv))
# plots
peff = None
pivs = None
pcal = None
prsd = None
# interactively adjust fit
if self.config.instrument.plot_level >= 1:
done = False
while not done:
yran = [np.min(100. * af / area), np.max(100. * af / area)]
effmax = np.nanmax(100. * fearea / area)
effmean = np.nanmean(100. * fearea / area)
peff = figure(title=self.action.args.plotlabel + ' Efficiency',
x_axis_label='Wave (A)',
y_axis_label='Effective Efficiency (%)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
peff.line(wf,
100. * af / area,
line_color='black',
legend_label='Data')
peff.line(w,
100. * fearea / area,
line_color='red',
legend_label='Fit')
peff.scatter(ww, ef, marker='x', legend_label='Rej')
peff.line([wlm0, wlm0],
yran,
line_color='green',
legend_label='WL lims')
peff.line([wlm1, wlm1], yran, line_color='green')
peff.line([wall0, wall1], [effmax, effmax],
line_color='black',
line_dash='dashed')
peff.line([wall0, wall1], [effmean, effmean],
line_color='black',
line_dash='dashdot')
set_plot_lims(peff, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(peff, self.context.bokeh_session)
if self.config.instrument.plot_level >= 1:
input("Next? <cr>: ")
else:
time.sleep(2. * self.config.instrument.plot_pause)
yran = [np.min(sf), np.max(sf)]
pivs = figure(title=self.action.args.plotlabel +
' Inverse sensitivity',
x_axis_label='Wave (A)',
y_axis_label='Invserse Sensitivity (Flux/e-/s)',
y_axis_type='log',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
pivs.line(wf, sf, line_color='black', legend_label='Data')
pivs.line(w, finvsen, line_color='red', legend_label='Fit')
pivs.scatter(ww, mf, marker='x', legend_label='Rej')
pivs.line([wlm0, wlm0],
yran,
line_color='green',
legend_label='WL lims')
pivs.line([wlm1, wlm1], yran, line_color='green')
set_plot_lims(pivs, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(pivs, self.context.bokeh_session)
if self.config.instrument.plot_level >= 1:
input("Next? <cr>: ")
else:
time.sleep(2. * self.config.instrument.plot_pause)
yran = [np.min(calspec[wl_good]), np.max(calspec[wl_good])]
pcal = figure(title=self.action.args.plotlabel + ' Calibrated',
x_axis_label='Wave (A)',
y_axis_label='Flux (ergs/s/cm^2/A)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
pcal.line(w, calspec, line_color='black', legend_label='Obs')
pcal.line(w, rsflx, line_color='red', legend_label='Ref')
pcal.line([wlm0, wlm0],
yran,
line_color='green',
legend_label='WL lims')
pcal.line([wlm1, wlm1], yran, line_color='green')
set_plot_lims(pcal, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(pcal, self.context.bokeh_session)
if self.config.instrument.plot_level >= 1:
input("Next? <cr>: ")
else:
time.sleep(2. * self.config.instrument.plot_pause)
yran = [np.min(resid), np.max(resid)]
prsd = figure(title=self.action.args.plotlabel +
' Residuals = %.1f +- %.1f (%%)' %
(rsd_mean, rsd_stdv),
x_axis_label='Wave (A)',
y_axis_label='Obs - Ref / Ref (%)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
prsd.line(wf,
resid,
line_color='black',
legend_label='Obs - Ref (%)')
if len(not_used) > 0:
prsd.scatter(ww, rbad, marker='x', legend_label='Rej')
prsd.line([wlm0, wlm0],
yran,
line_color='green',
legend_label='WL lims')
prsd.line([wlm1, wlm1], yran, line_color='green')
prsd.line([wall0, wall1], [rsd_mean, rsd_mean],
line_color='red')
prsd.line([wall0, wall1],
[rsd_mean + rsd_stdv, rsd_mean + rsd_stdv],
line_color='black',
line_dash='dashed')
prsd.line([wall0, wall1],
[rsd_mean - rsd_stdv, rsd_mean - rsd_stdv],
line_color='black',
line_dash='dashed')
set_plot_lims(prsd, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(prsd, self.context.bokeh_session)
if self.config.instrument.plot_level >= 1:
qstr = input("Current fit order = %d, "
"New fit order? <int>, <cr> - done: " % ford)
if len(qstr) <= 0:
done = True
else:
try:
ford = int(qstr)
# update fit of inverse sensitivity
res = np.polyfit(wf - wf0, sf, deg=ford, w=mw)
finvsen = np.polyval(res, w - wf0)
sinvsen = np.polyval(res, wf - wf0)
calspec = obsspec * finvsen
scalspec = bf * sinvsen
# update polynomial fit of effective area
res = np.polyfit(wf - wf0, af, ford, w=mw)
fearea = np.polyval(res, w - wf0)
# re-calculate residuals
resid = 100.0 * (scalspec - rf) / rf
if len(not_used) > 0:
rbad = resid[not_used]
else:
rbad = None
rsd_mean = float(np.nanmean(resid[used]))
rsd_stdv = float(np.nanstd(resid[used]))
self.logger.info(
"Calibration residuals = %f +- %f %%" %
(rsd_mean, rsd_stdv))
except ValueError:
print("Bad fit order, try again")
else:
done = True
time.sleep(2. * self.config.instrument.plot_pause)
# log results
effmax = float(np.nanmax(100. * fearea / area))
effmean = float(np.nanmean(100. * fearea / area))
self.logger.info("Peak, mean efficiency (%%): %.1f, %.1f" %
(effmax, effmean))
self.logger.info("Fit order = %d" % ford)
# output plots
pfname = "std_%05d_%s_%s_%s_%d" % (
self.action.args.ccddata.header['FRAMENO'], stdname,
self.action.args.grating.strip(),
self.action.args.ifuname.strip(), int(self.action.args.cwave))
# Save plots
save_plot(peff, filename=pfname + '_eff.png') # Efficiency
save_plot(pivs, filename=pfname + '_invsens.png') # Inv. Sens.
save_plot(prsd, filename=pfname + '_resid.png') # Residuals
save_plot(pcal, filename=pfname + '_cal.png') # Calibrated
log_string = MakeInvsens.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
# write out effective inverse sensitivity
# update inverse sensitivity header
hdr = self.action.args.ccddata.header.copy()
hdr['HISTORY'] = log_string
hdr['INVSENS'] = (True, 'effective inv. sens. spectrum?')
hdr['INVSW0'] = (w[z0], 'low wavelength for eff. inv. sens.')
hdr['INVSW1'] = (w[z1], 'high wavelength for eff. inv. sens.')
hdr['INVSZ0'] = (z0, 'low wave pixel for eff. inv. sens.')
hdr['INVSZ1'] = (z1, 'high wave pixel for eff. inv. sens.')
hdr['INVSY0'] = (gy0, 'low spatial pixel for eff. inv. sens.')
hdr['INVSY1'] = (gy1, 'high spatial pixel for eff. inv. sens.')
hdr['INVSLMX'] = (mxsl, 'brightest std star slice')
hdr['INVSL0'] = (sl0, 'lowest std star slice summed')
hdr['INVSL1'] = (sl1, 'highest std star slice summed')
hdr['INVSLY'] = (cy, 'spatial pixel position of std within slice')
hdr['INVFW0'] = (wlm0, 'low wavelength for fits')
hdr['INVFW1'] = (wlm1, 'high wavelength for fits')
hdr['INVFORD'] = (ford, 'fit order')
hdr['EXPTIME'] = (1., 'effective exposure time (seconds)')
hdr['XPOSURE'] = (1., 'effective exposure time (seconds)')
# remove old WCS
del hdr['RADESYS']
del hdr['EQUINOX']
del hdr['LONPOLE']
del hdr['LATPOLE']
del hdr['NAXIS2']
if 'NAXIS3' in hdr:
del hdr['NAXIS3']
del hdr['CTYPE1']
del hdr['CTYPE2']
del hdr['CTYPE3']
del hdr['CUNIT1']
del hdr['CUNIT2']
del hdr['CUNIT3']
del hdr['CNAME1']
del hdr['CNAME2']
del hdr['CNAME3']
del hdr['CRVAL1']
del hdr['CRVAL2']
del hdr['CRVAL3']
del hdr['CRPIX1']
del hdr['CRPIX2']
del hdr['CRPIX3']
del hdr['CD1_1']
del hdr['CD1_2']
del hdr['CD2_1']
del hdr['CD2_2']
del hdr['CD3_3']
# set wavelength axis WCS values
hdr['WCSDIM'] = 1
hdr['CTYPE1'] = ('AWAV', 'Air Wavelengths')
hdr['CUNIT1'] = ('Angstrom', 'Wavelength units')
hdr['CNAME1'] = ('KCWI INVSENS Wavelength', 'Wavelength name')
hdr['CRVAL1'] = (w0, 'Wavelength zeropoint')
hdr['CRPIX1'] = (crpixw, 'Wavelength reference pixel')
hdr['CDELT1'] = (dw, 'Wavelength Angstroms per pixel')
ofn = self.action.args.ccddata.header['OFNAME']
invsname = ofn.split('.fits')[0] + '_' + suffix + '.fits'
eaname = ofn.split('.fits')[0] + '_ea.fits'
# set units
invsens_u = u.erg / (u.angstrom * u.cm**2 * u.s * u.electron)
# output inverse sensitivity
out_invsens = CCDData(np.asarray([invsen, finvsen, obsspec]),
meta=hdr,
unit=invsens_u)
kcwi_fits_writer(out_invsens,
output_file=invsname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=out_invsens,
suffix=suffix,
newtype='INVSENS')
# output effective area
ea_u = u.cm**2 / u.angstrom
out_ea = CCDData(np.asarray([earea, fearea]), meta=hdr, unit=ea_u)
kcwi_fits_writer(out_ea,
output_file=eaname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=out_ea,
suffix='ea',
newtype='EAREA')
return self.action.args
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation
"""
method = 'average'
suffix = self.action.args.stack_type.lower()
self.logger.info("Stacking flats using method %s" % method)
combine_list = list(self.combine_list['filename'])
# get flat stack output name
stname = strip_fname(combine_list[-1]) + '_' + suffix + '.fits'
stack = []
stackf = []
mask = None
for flat in combine_list:
# get flat intensity (int) image file name in redux directory
stackf.append(strip_fname(flat) + '_intd.fits')
flatfn = os.path.join(self.config.instrument.cwd,
self.config.instrument.output_directory,
stackf[-1])
# using [0] gets just the image data
f = kcwi_fits_reader(flatfn)[0]
# Set mask to None to prevent ccdproc.combine from masking
f.mask = None
stack.append(f)
stacked = ccdproc.combine(stack, method=method, sigma_clip=True,
sigma_clip_low_thresh=None,
sigma_clip_high_thresh=2.0)
# Get the BPM out of one of the flats (bpm is the same for all)
# and add it to the stacked flat as the stack's mask
last_flat_name = strip_fname(combine_list[-1]) + '_intd.fits'
last_flat_path = os.path.join(self.config.instrument.cwd,
self.config.instrument.output_directory,
last_flat_name)
last_flat = kcwi_fits_reader(last_flat_path)[0]
stacked.mask = last_flat.mask
stacked.header['IMTYPE'] = self.action.args.stack_type
stacked.header['NSTACK'] = (len(combine_list),
'number of images stacked')
stacked.header['STCKMETH'] = (method, 'method used for stacking')
for ii, fname in enumerate(stackf):
stacked.header['STACKF%d' % (ii + 1)] = (fname, "stack input file")
log_string = StackFlats.__module__
stacked.header['HISTORY'] = log_string
# output stacked flat
kcwi_fits_writer(stacked, output_file=stname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=stacked, suffix=suffix,
newtype=self.action.args.stack_type,
filename=self.action.args.name)
self.context.proctab.write_proctab()
self.logger.info(log_string)
return self.action.args
def _perform(self):
# Header keyword to update
key = 'IMGRECT'
keycom = 'Image rectified?'
# get amp mode
ampmode = self.action.args.ccddata.header['AMPMODE'].strip().upper()
if '__B' in ampmode or '__G' in ampmode:
newimg = np.rot90(self.action.args.ccddata.data, 2)
self.action.args.ccddata.data = newimg
if self.action.args.ccddata.uncertainty:
newunc = np.rot90(self.action.args.ccddata.uncertainty.array,
2)
self.action.args.ccddata.uncertainty.array = newunc
mask = getattr(self.action.args.ccddata, "mask", None)
if mask is not None:
newmask = np.rot90(mask, 2)
self.action.args.ccddata.mask = newmask
else:
self.logger.info("No mask data to rectify")
flags = getattr(self.action.args.ccddata, "flags", None)
if flags is not None:
newflags = np.rot90(flags, 2)
self.action.args.ccddata.flags = newflags
else:
self.logger.info("No flags data to rectify")
elif '__D' in ampmode or '__F' in ampmode:
newimg = np.fliplr(self.action.args.ccddata.data)
self.action.args.ccddata.data = newimg
if self.action.args.ccddata.uncertainty:
newunc = np.fliplr(self.action.args.ccddata.uncertainty.array)
self.action.args.ccddata.uncertainty.array = newunc
mask = getattr(self.action.args.ccddata, "mask", None)
if mask is not None:
newmask = np.fliplr(mask)
self.action.args.ccddata.mask = newmask
else:
self.logger.info("No mask data to rectify")
flags = getattr(self.action.args.ccddata, "flags", None)
if flags is not None:
newflags = np.fliplr(flags)
self.action.args.ccddata.flags = newflags
else:
self.logger.info("No flags data to rectify")
elif '__A' in ampmode or '__H' in ampmode or 'TUP' in ampmode:
newimg = np.flipud(self.action.args.ccddata.data)
self.action.args.ccddata.data = newimg
if self.action.args.ccddata.uncertainty:
newunc = np.flipud(self.action.args.ccddata.uncertainty.array)
self.action.args.ccddata.uncertainty.array = newunc
mask = getattr(self.action.args.ccddata, "mask", None)
if mask is not None:
newmask = np.flipud(mask)
self.action.args.ccddata.mask = newmask
else:
self.logger.info("No mask data to rectify")
flags = getattr(self.action.args.ccddata, "flags", None)
if flags is not None:
newflags = np.flipud(flags)
self.action.args.ccddata.flags = newflags
else:
self.logger.info("No flags data to rectify")
self.action.args.ccddata.header[key] = (True, keycom)
log_string = RectifyImage.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
# write out int image
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="int")
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix="int")
self.context.proctab.write_proctab()
return self.action.args
def _perform(self):
self.logger.info("Creating 2D image from 3D cube")
log_string = CubeImage.__module__
# get wavelength ranges
wall0 = self.action.args.ccddata.header['WAVALL0']
wall1 = self.action.args.ccddata.header['WAVALL1']
w0 = self.action.args.ccddata.header['CRVAL3']
dw = self.action.args.ccddata.header['CD3_3']
crpixw = self.action.args.ccddata.header['CRPIX3']
y0 = int((wall0 - w0) / dw)
if y0 < 0:
y0 = 0
out_w0 = y0 * dw + w0
y1 = int((wall1 - w0) / dw)
if y1 > self.action.args.cube_size[0]:
y1 = self.action.args.cube_size[0]
out_y = y1 - y0
# create output image
cub_x = self.action.args.cube_size[1]
out_x = int(24 * cub_x)
out_img = np.zeros((out_y, out_x), dtype=np.float)
# set spatial scale
s0 = 0.
ds = 24.0 / out_x
crpixs = 1.
self.logger.info("cube dims: y, x, z: %d, %d, %d" %
self.action.args.cube_size)
self.logger.info("y0, y1 = %d, %d: out_x, out_y = %d, %d" %
(y0, y1, out_x, out_y))
# loop over slices
for isl in range(24):
x0 = isl * cub_x
x1 = (isl + 1) * cub_x
out_img[:, x0:x1] = self.action.args.ccddata.data[y0:y1, :, isl]
# output CCDData structure
out_ccd = CCDData(out_img,
meta=self.action.args.ccddata.header,
unit=self.action.args.ccddata.unit)
# update header
out_ccd.header['HISTORY'] = log_string
del out_ccd.header['RADESYS']
del out_ccd.header['LONPOLE']
del out_ccd.header['LATPOLE']
del out_ccd.header['CTYPE1']
del out_ccd.header['CTYPE2']
del out_ccd.header['CTYPE3']
del out_ccd.header['CUNIT1']
del out_ccd.header['CUNIT2']
del out_ccd.header['CUNIT3']
del out_ccd.header['CNAME1']
del out_ccd.header['CNAME2']
del out_ccd.header['CNAME3']
del out_ccd.header['CRVAL1']
del out_ccd.header['CRVAL2']
del out_ccd.header['CRVAL3']
del out_ccd.header['CRPIX1']
del out_ccd.header['CRPIX2']
del out_ccd.header['CRPIX3']
del out_ccd.header['CD1_1']
del out_ccd.header['CD1_2']
del out_ccd.header['CD2_1']
del out_ccd.header['CD2_2']
del out_ccd.header['CD3_3']
out_ccd.header['WCSDIM'] = 2
out_ccd.header['CTYPE1'] = ('SPATIAL', 'slice number')
out_ccd.header['CUNIT1'] = ('slu', 'slice units')
out_ccd.header['CNAME1'] = ('KCWI Slice', 'slice name')
out_ccd.header['CRVAL1'] = (s0, 'slice zeropoint')
out_ccd.header['CRPIX1'] = (crpixs, 'slice reference pixel')
out_ccd.header['CDELT1'] = (ds, 'slice units per pixel')
out_ccd.header['CTYPE2'] = ('AWAV', 'Air Wavelengths')
out_ccd.header['CUNIT2'] = ('Angstrom', 'Wavelength units')
out_ccd.header['CNAME2'] = ('KCWI Wavelength', 'Wavelength name')
out_ccd.header['CRVAL2'] = (out_w0, 'Wavelength zeropoint')
out_ccd.header['CRPIX2'] = (crpixw, 'Wavelength reference pixel')
out_ccd.header['CDELT2'] = (dw, 'Wavelength Angstroms per pixel')
# write out image
kcwi_fits_writer(out_ccd,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="icube_2d")
self.logger.info(log_string)
return self.action.args
def _perform(self):
"""Correct for differential atmospheric refraction"""
self.logger.info("Correcting for DAR")
# image size
image_size = self.action.args.ccddata.data.shape
# get wavelengths
w0 = self.action.args.ccddata.header['CRVAL3']
dw = self.action.args.ccddata.header['CD3_3']
waves = w0 + np.arange(image_size[0]) * dw
wgoo0 = self.action.args.ccddata.header['WAVGOOD0']
wgoo1 = self.action.args.ccddata.header['WAVGOOD1']
wref = self.action.args.ccddata.header['WAVMID']
self.logger.info("Ref WL = %.1f, good WL range = (%.1f - %.1f" %
(wref, wgoo0, wgoo1))
# spatial scales in arcsec/item
y_scale = self.action.args.ccddata.header['PXSCL'] * 3600.
x_scale = self.action.args.ccddata.header['SLSCL'] * 3600.
# padding depends on grating
if 'H' in self.action.args.grating:
padding_as = 2.0
elif 'M' in self.action.args.grating:
padding_as = 3.0
else:
padding_as = 4.0
padding_x = int(padding_as / x_scale)
padding_y = int(padding_as / y_scale)
# update WCS
crpix1 = self.action.args.ccddata.header['CRPIX1']
crpix2 = self.action.args.ccddata.header['CRPIX2']
self.action.args.ccddata.header['CRPIX1'] = crpix1 + float(padding_x)
self.action.args.ccddata.header['CRPIX2'] = crpix2 + float(padding_y)
# airmass
airmass = self.action.args.ccddata.header['AIRMASS']
self.logger.info("Airmass: %.3f" % airmass)
# IFU orientation
ifu_pa = self.action.args.ccddata.header['IFUPA']
# Parallactic angle
parallactic_angle = self.action.args.ccddata.header['PARANG']
# Projection angle in radians
projection_angle_deg = ifu_pa - parallactic_angle
projection_angle = math.radians(projection_angle_deg)
self.logger.info("DAR Angles: ifu_pa, parang, projang (deg): "
"%.2f, %.2f, %.2f" %
(ifu_pa, parallactic_angle, projection_angle_deg))
# dispersion over goo wl range in arcsec
dispersion_max_as = atm_disper(wgoo1, wgoo0, airmass)
# projected onto IFU
xdmax_as = dispersion_max_as * math.sin(projection_angle)
ydmax_as = dispersion_max_as * math.cos(projection_angle)
self.logger.info("DAR over GOOD WL range: total, x, y (asec): "
"%.2f, %.2f, %.2f" %
(dispersion_max_as, xdmax_as, ydmax_as))
# now in pixels
xdmax_px = xdmax_as / x_scale
ydmax_px = ydmax_as / y_scale
dmax_px = math.sqrt(xdmax_px**2 + ydmax_px**2)
self.logger.info("DAR over GOOD WL range: total, x, y (pix): "
"%.2f, %.2f, %.2f" % (dmax_px, xdmax_px, ydmax_px))
# prepare output cubes
output_image = np.zeros((image_size[0], image_size[1] + 2 * padding_y,
image_size[2] + 2 * padding_x),
dtype=np.float64)
output_stddev = output_image.copy()
output_mask = np.zeros((image_size[0], image_size[1] + 2 * padding_y,
image_size[2] + 2 * padding_x),
dtype=np.uint8)
output_flags = np.zeros((image_size[0], image_size[1] + 2 * padding_y,
image_size[2] + 2 * padding_x),
dtype=np.uint8)
# DAR padded pixel flag
output_flags += 128
output_image[:, padding_y:(padding_y+image_size[1]),
padding_x:(padding_x+image_size[2])] = \
self.action.args.ccddata.data
output_stddev[:, padding_y:(padding_y+image_size[1]),
padding_x:(padding_x+image_size[2])] = \
self.action.args.ccddata.uncertainty.array
output_mask[:, padding_y:(padding_y+image_size[1]),
padding_x:(padding_x+image_size[2])] = \
self.action.args.ccddata.mask
output_flags[:, padding_y:(padding_y+image_size[1]),
padding_x:(padding_x+image_size[2])] = \
self.action.args.ccddata.flags
# check for obj, sky cubes
output_obj = None
output_sky = None
if self.action.args.nasmask and self.action.args.numopen > 1:
ofn = self.action.args.name
objfn = strip_fname(ofn) + '_ocube.fits'
full_path = os.path.join(self.config.instrument.cwd,
self.config.instrument.output_directory,
objfn)
if os.path.exists(full_path):
obj = kcwi_fits_reader(full_path)[0]
output_obj = np.zeros(
(image_size[0], image_size[1] + 2 * padding_y,
image_size[2] + 2 * padding_x),
dtype=np.float64)
output_obj[:, padding_y:(padding_y + image_size[1]),
padding_x:(padding_x + image_size[2])] = obj.data
skyfn = strip_fname(ofn) + '_scube.fits'
full_path = os.path.join(self.config.instrument.cwd,
self.config.instrument.output_directory,
skyfn)
if os.path.exists(full_path):
sky = kcwi_fits_reader(full_path)[0]
output_sky = np.zeros(
(image_size[0], image_size[1] + 2 * padding_y,
image_size[2] + 2 * padding_x),
dtype=np.float64)
output_sky[:, padding_y:(padding_y + image_size[1]),
padding_x:(padding_x + image_size[2])] = sky.data
# check if we have a standard star observation
output_del = None
stdfile, _ = kcwi_get_std(self.action.args.ccddata.header['OBJECT'],
self.logger)
if stdfile is not None:
afn = self.action.args.ccddata.header['ARCFL']
delfn = strip_fname(afn) + '_dcube.fits'
full_path = os.path.join(self.config.instrument.cwd,
self.config.instrument.output_directory,
delfn)
if os.path.exists(full_path):
dew = kcwi_fits_reader(full_path)[0]
output_del = np.zeros(
(image_size[0], image_size[1] + 2 * padding_y,
image_size[2] + 2 * padding_x),
dtype=np.float64)
output_del[:, padding_y:(padding_y + image_size[1]),
padding_x:(padding_x + image_size[2])] = dew.data
# Perform correction
for j, wl in enumerate(waves):
dispersion_correction = atm_disper(wref, wl, airmass)
x_shift = dispersion_correction * \
math.sin(projection_angle) / x_scale
y_shift = dispersion_correction * \
math.cos(projection_angle) / y_scale
output_image[j, :, :] = shift(output_image[j, :, :],
(y_shift, x_shift))
output_stddev[j, :, :] = shift(output_stddev[j, :, :],
(y_shift, x_shift))
output_mask[j, :, :] = shift(output_mask[j, :, :],
(y_shift, x_shift))
output_flags[j, :, :] = shift(output_flags[j, :, :],
(y_shift, x_shift))
# for obj, sky if they exist
if output_obj is not None:
for j, wl in enumerate(waves):
dispersion_correction = atm_disper(wref, wl, airmass)
x_shift = dispersion_correction * \
math.sin(projection_angle) / x_scale
y_shift = dispersion_correction * \
math.cos(projection_angle) / y_scale
output_obj[j, :, :] = shift(output_obj[j, :, :],
(y_shift, x_shift))
if output_sky is not None:
for j, wl in enumerate(waves):
dispersion_correction = atm_disper(wref, wl, airmass)
x_shift = dispersion_correction * \
math.sin(projection_angle) / x_scale
y_shift = dispersion_correction * \
math.cos(projection_angle) / y_scale
output_sky[j, :, :] = shift(output_sky[j, :, :],
(y_shift, x_shift))
# for delta wavelength cube, if it exists
if output_del is not None:
for j, wl in enumerate(waves):
dispersion_correction = atm_disper(wref, wl, airmass)
x_shift = dispersion_correction * \
math.sin(projection_angle) / x_scale
y_shift = dispersion_correction * \
math.cos(projection_angle) / y_scale
output_del[j, :, :] = shift(output_del[j, :, :],
(y_shift, x_shift))
self.action.args.ccddata.data = output_image
self.action.args.ccddata.uncertainty.array = output_stddev
self.action.args.ccddata.mask = output_mask
self.action.args.ccddata.flags = output_flags
log_string = CorrectDar.__module__
# update header
self.action.args.ccddata.header['HISTORY'] = log_string
self.action.args.ccddata.header['DARCOR'] = (True, 'DAR corrected?')
self.action.args.ccddata.header['DARANG'] = (projection_angle_deg,
'DAR projection angle '
'(deg)')
self.action.args.ccddata.header['DARPADX'] = (padding_x,
'DAR X padding (pix)')
self.action.args.ccddata.header['DARPADY'] = (padding_y,
'DAR Y padding (pix)')
self.action.args.ccddata.header['DAREFWL'] = (wref,
'DAR reference wl (Ang)')
# write out corrected image
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="icubed")
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix="icubed",
filename=self.action.args.name)
self.context.proctab.write_proctab()
# check for sky, obj cube
if output_obj is not None:
out_obj = CCDData(output_obj,
meta=self.action.args.ccddata.header,
unit=self.action.args.ccddata.unit)
kcwi_fits_writer(
out_obj,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="ocubed")
if output_sky is not None:
out_sky = CCDData(output_sky,
meta=self.action.args.ccddata.header,
unit=self.action.args.ccddata.unit)
kcwi_fits_writer(
out_sky,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="scubed")
# check for delta wave cube
if output_del is not None:
out_del = CCDData(output_del,
meta=self.action.args.ccddata.header,
unit=self.action.args.ccddata.unit)
kcwi_fits_writer(
out_del,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="dcubed")
self.logger.info(log_string)
return self.action.args
def _perform(self):
# Header keyword to update
key = 'STDCOR'
keycom = 'std corrected?'
target_type = 'INVSENS'
obj = None
sky = None
self.logger.info("Calibrating object flux")
tab = self.context.proctab.n_proctab(frame=self.action.args.ccddata,
target_type=target_type,
nearest=True)
self.logger.info("%d invsens files found" % len(tab))
if self.action.args.invsname is not None:
# read in master calibration (inverse sensitivity)
invsname = self.action.args.invsname
self.logger.info("Reading invsens: %s" % invsname)
hdul = pf.open(
os.path.join(os.path.dirname(self.action.args.name), 'redux',
invsname))
mcal = hdul[0].data[1, :]
mchdr = hdul[0].header
hdul.close()
# get dimensions
mcsz = mcal.shape
# get master std waves
mcw0 = mchdr['CRVAL1']
mcdw = mchdr['CDELT1']
mcwav = mcw0 + np.arange(mcsz[0]) * mcdw
# get master std image number
msimgno = mchdr['FRAMENO']
# get input object image dimensions
sz = self.action.args.ccddata.data.shape
# get object waves
w0 = self.action.args.ccddata.header['CRVAL3']
dw = self.action.args.ccddata.header['CD3_3']
wav = w0 + np.arange(sz[0]) * dw
# get exposure time
expt = self.action.args.ccddata.header['XPOSURE']
# resample onto object waves, if needed
if w0 != mcw0 or dw != mcdw or wav[-1] != mcwav[-1] or \
sz[0] != mcsz[0]:
self.logger.warning("wavelength scales not identical, "
"resampling standard")
print(w0, mcw0, dw, mcdw, wav[-1], mcwav[-1], sz[0], mcsz[0])
mcint = interp1d(mcwav,
mcal,
kind='cubic',
fill_value='extrapolate')
mscal = mcint(wav) * 1.e16 / expt
else:
mscal = mcal * 1.e16 / expt
# extinction correct calibration
kcwi_correct_extin(mscal,
self.action.args.ccddata.header,
logger=self.logger)
# do calibration
for isl in range(sz[2]):
for ix in range(sz[1]):
self.action.args.ccddata.data[:, ix, isl] *= mscal
self.action.args.ccddata.uncertainty.array[:, ix, isl] *= \
mscal ** 2
# check for obj, sky cubes
if self.action.args.nasmask and self.action.args.numopen > 1:
ofn = self.action.args.ccddata.header['OFNAME']
# obj cube
objfn = ofn.split('.')[0] + '_ocubed.fits'
full_path = os.path.join(
os.path.dirname(self.action.args.name),
self.config.instrument.output_directory, objfn)
if os.path.exists(full_path):
obj = kcwi_fits_reader(full_path)[0]
# do calibration
for isl in range(sz[2]):
for ix in range(sz[1]):
obj.data[:, ix, isl] *= mscal
# sky cube
skyfn = ofn.split('.')[0] + '_scubed.fits'
full_path = os.path.join(
os.path.dirname(self.action.args.name),
self.config.instrument.output_directory, skyfn)
if os.path.exists(full_path):
sky = kcwi_fits_reader(full_path)[0]
# do calibration
for isl in range(sz[2]):
for ix in range(sz[1]):
sky.data[:, ix, isl] *= mscal
# units
flam16_u = 1.e16 * u.erg / (u.angstrom * u.cm**2 * u.s)
self.action.args.ccddata.unit = flam16_u
self.action.args.ccddata.uncertainty.unit = flam16_u
if obj is not None:
obj.unit = flam16_u
if sky is not None:
sky.unit = flam16_u
# update header keywords
self.action.args.ccddata.header[key] = (True, keycom)
self.action.args.ccddata.header['MSFILE'] = (invsname,
"Master std filename")
self.action.args.ccddata.header['MSIMNO'] = (
msimgno, 'master std image number')
else:
self.action.args.ccddata.header[key] = (False, keycom)
log_string = FluxCalibrate.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
# write out icubes image
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="icubes")
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix="icubes")
self.context.proctab.write_proctab()
# check for sky, obj cube
if obj is not None:
out_obj = CCDData(obj,
meta=self.action.args.ccddata.header,
unit=self.action.args.ccddata.unit)
kcwi_fits_writer(
out_obj,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="ocubes")
if sky is not None:
out_sky = CCDData(sky,
meta=self.action.args.ccddata.header,
unit=self.action.args.ccddata.unit)
kcwi_fits_writer(
out_sky,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="scubes")
self.logger.info(log_string)
return self.action.args
def _perform(self):
self.logger.info("Subtracting sky background")
# Header keyword to update
key = 'SKYCOR'
keycom = 'sky corrected?'
target_type = 'SKY'
skyfile = self.action.args.skyfile
skymask = self.action.args.skymask
if not self.action.args.skyfile:
tab = self.context.proctab.search_proctab(
frame=self.action.args.ccddata,
target_type=target_type,
nearest=True)
self.logger.info("%d master sky frames found" % len(tab))
if len(tab) > 0:
skyfile = tab['filename'][0]
msname = strip_fname(skyfile) + '_' + target_type.lower() + ".fits"
if os.path.exists(
os.path.join(self.config.instrument.cwd, 'redux', msname)):
self.logger.info("Reading image: %s" % msname)
msky = kcwi_fits_reader(
os.path.join(self.config.instrument.cwd, 'redux', msname))[0]
# scale the sky?
obtime = self.action.args.ccddata.header['XPOSURE']
sktime = msky.header['XPOSURE']
if obtime <= 0. or sktime <= 0.:
self.logger.warning(
"Bad exposure times (obj, sky): %.1f, %1f" %
(obtime, sktime))
skscl = 1.
else:
skscl = obtime / sktime
self.logger.info("Sky scale factor = %.3f" % skscl)
# do the subtraction
self.action.args.ccddata.data -= msky.data * skscl
# update header keywords
self.action.args.ccddata.header[key] = (True, keycom)
self.action.args.ccddata.header['SKYMAST'] = (
msname, "Master sky filename")
self.action.args.ccddata.header['SKYSCL'] = (skscl,
'sky scale factor')
if skymask:
self.action.args.ccddata.header['SKYMSKF'] = (skymask,
'sky mask file')
else:
# update header keywords
self.action.args.ccddata.header[key] = (False, keycom)
log_string = SubtractSky.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
# write out int image
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="intk")
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix="intk",
filename=self.action.args.name)
self.context.proctab.write_proctab()
self.logger.info(log_string)
return self.action.args
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation
"""
self.logger.info("Creating master illumination correction")
suffix = self.action.args.new_type.lower()
insuff = self.action.args.stack_type.lower()
stack_list = list(self.stack_list['filename'])
if len(stack_list) <= 0:
self.logger.warning("No flats found!")
return self.action.args
# get root for maps
tab = self.context.proctab.search_proctab(
frame=self.action.args.ccddata, target_type='ARCLAMP',
target_group=self.action.args.groupid)
if len(tab) <= 0:
self.logger.error("Geometry not solved!")
return self.action.args
mroot = strip_fname(tab['filename'][-1])
# Wavelength map image
wmf = mroot + '_wavemap.fits'
self.logger.info("Reading image: %s" % wmf)
wavemap = kcwi_fits_reader(
os.path.join(self.config.instrument.cwd, 'redux',
wmf))[0]
# Slice map image
slf = mroot + '_slicemap.fits'
self.logger.info("Reading image: %s" % slf)
slicemap = kcwi_fits_reader(os.path.join(
self.config.instrument.cwd, 'redux',
slf))[0]
# Position map image
pof = mroot + '_posmap.fits'
self.logger.info("Reading image: %s" % pof)
posmap = kcwi_fits_reader(os.path.join(
self.config.instrument.cwd, 'redux',
pof))[0]
# Read in stacked flat image
stname = strip_fname(stack_list[0]) + '_' + insuff + '.fits'
self.logger.info("Reading image: %s" % stname)
stacked = kcwi_fits_reader(os.path.join(
self.config.instrument.cwd, 'redux',
stname))[0]
# get type of flat
internal = ('SFLAT' in stacked.header['IMTYPE'])
twiflat = ('STWIF' in stacked.header['IMTYPE'])
domeflat = ('SDOME' in stacked.header['IMTYPE'])
if internal:
self.logger.info("Internal Flat")
elif twiflat:
self.logger.info("Twilight Flat")
elif domeflat:
self.logger.info("Dome Flat")
else:
self.logger.error("Flat of Unknown Type!")
return self.action.args
# knots per pixel
knotspp = self.config.instrument.KNOTSPP
# get image size
ny = stacked.header['NAXIS2']
# get binning
xbin = self.action.args.xbinsize
# Parameters for fitting
# vignetted slice position range
fitl = int(4/xbin)
fitr = int(24/xbin)
# un-vignetted slice position range
flatl = int(34/xbin)
flatr = int(72/xbin)
# flat fitting slice position range
ffleft = int(10/xbin)
ffright = int(70/xbin)
nrefx = int(ffright - ffleft)
buffer = 6.0/float(xbin)
# reference slice
refslice = 9
allidx = np.arange(int(140/xbin))
newflat = stacked.data.copy()
# dichroic fraction
try:
dichroic_fraction = wavemap.header['DICHFRAC']
except KeyError:
dichroic_fraction = 1.
# get reference slice data
q = [i for i, v in enumerate(slicemap.data.flat) if v == refslice]
# get wavelength limits
waves = wavemap.data.compress((wavemap.data > 0.).flat)
waves = [waves.min(), waves.max()]
self.logger.info("Wavelength limits: %.1f - %1.f" % (waves[0],
waves[1]))
# correct vignetting if we are using internal flats
if internal:
self.logger.info("Internal flats require vignetting correction")
# get good region for fitting
if self.action.args.camera == 0: # Blue
wmin = waves[0]
wmax = min([waves[1], 5620.])
elif self.action.args.camera == 1: # Red
wmin = max([waves[0], 5580.])
wmax = waves[1]
else:
self.logger.warning("Camera keyword not defined")
wmin = waves[0]
wmax = waves[1]
dw = (wmax - wmin) / 30.0
wavemin = (wmin+wmax) / 2.0 - dw
wavemax = (wmin+wmax) / 2.0 + dw
self.logger.info("Using %.1f - %.1f A of slice %d" % (wavemin,
wavemax,
refslice))
xflat = []
yflat = []
wflat = []
qq = []
for i in q:
if wavemin < wavemap.data.flat[i] < wavemax:
xflat.append(posmap.data.flat[i])
yflat.append(stacked.data.flat[i])
wflat.append(wavemap.data.flat[i])
qq.append(i)
# get un-vignetted portion
qflat = [i for i, v in enumerate(xflat) if flatl <= v <= flatr]
xflat = [xflat[i] for i in qflat]
yflat = [yflat[i] for i in qflat]
wflat = [wflat[i] for i in qflat]
# sort on wavelength
sw = np.argsort(wflat)
ywflat = [yflat[i] for i in sw]
wwflat = [wflat[i] for i in sw]
ww0 = np.min(wwflat)
# fit wavelength slope
wavelinfit = np.polyfit(wwflat-ww0, ywflat, 2)
wslfit = np.polyval(wavelinfit, wflat-ww0)
# plot slope fit
if self.config.instrument.plot_level >= 1:
p = figure(title=self.action.args.plotlabel + ' WAVE SLOPE FIT',
x_axis_label='wave px',
y_axis_label='counts',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(wwflat, ywflat, legend_label="Data")
p.line(wflat, wslfit, line_color='red', line_width=3,
legend_label="Fit")
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# take out slope
yflat = yflat / wslfit
# now sort on slice position
ss = np.argsort(xflat)
xflat = [xflat[i] for i in ss]
yflat = [yflat[i] for i in ss]
# fit un-vignetted slope
resflat = np.polyfit(xflat, yflat, 1)
# select the points we will fit for the vignetting
# get reference region
xfit = [posmap.data.flat[i] for i in qq]
yfit = [stacked.data.flat[i] for i in qq]
wflat = [wavemap.data.flat[i] for i in qq]
# take out wavelength slope
yfit = yfit / np.polyval(wavelinfit, wflat-ww0)
# select the vignetted region
qfit = [i for i, v in enumerate(xfit) if fitl <= v <= fitr]
xfit = [xfit[i] for i in qfit]
yfit = [yfit[i] for i in qfit]
# sort on slice position
s = np.argsort(xfit)
xfit = [xfit[i] for i in s]
yfit = [yfit[i] for i in s]
# fit vignetted slope
resfit = np.polyfit(xfit, yfit, 1)
# corrected data
ycdata = stacked.data.flat[qq] / \
np.polyval(wavelinfit, wavemap.data.flat[qq]-ww0)
ycmin = 0.5 # np.min(ycdata)
ycmax = 1.25 # np.max(ycdata)
# compute the intersection
xinter = -(resflat[1] - resfit[1]) / (resflat[0] - resfit[0])
# plot slice profile and fits
if self.config.instrument.plot_level >= 1:
p = figure(title=self.action.args.plotlabel + ' Vignetting',
x_axis_label='Slice Pos (px)',
y_axis_label='Ratio',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(posmap.data.flat[qq], ycdata, legend_label='Data')
p.line(allidx, resfit[1] + resfit[0]*allidx,
line_color='purple', legend_label='Vign.')
p.line(allidx, resflat[1] + resflat[0]*allidx, line_color='red',
legend_label='UnVign.')
p.line([fitl, fitl], [ycmin, ycmax], line_color='blue')
p.line([fitr, fitr], [ycmin, ycmax], line_color='blue')
p.line([flatl, flatl], [ycmin, ycmax], line_color='green')
p.line([flatr, flatr], [ycmin, ycmax], line_color='green')
p.line([xinter-buffer, xinter-buffer], [ycmin, ycmax],
line_color='black')
p.line([xinter + buffer, xinter + buffer], [ycmin, ycmax],
line_color='black')
p.line([xinter, xinter], [ycmin, ycmax], line_color='red')
p.y_range = Range1d(ycmin, ycmax)
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# figure out where the correction applies
qcor = [i for i, v in enumerate(posmap.data.flat)
if 0 <= v <= (xinter-buffer)]
# apply the correction!
self.logger.info("Applying vignetting correction...")
for i in qcor:
newflat.flat[i] = (resflat[1]+resflat[0]*posmap.data.flat[i]) \
/ (resfit[1]+resfit[0]*posmap.data.flat[i]) * \
stacked.data.flat[i]
# now deal with the intermediate (buffer) region
self.logger.info("Done, now handling buffer region")
# get buffer points to fit in reference region
qbff = [i for i in qq if (xinter-buffer) <=
posmap.data.flat[i] <= (xinter+buffer)]
# get slice pos and data for buffer fitting
xbuff = [posmap.data.flat[i] for i in qbff]
ybuff = [stacked.data.flat[i] / np.polyval(wavelinfit,
wavemap.data.flat[i]-ww0)
for i in qbff]
# sort on slice position
ssp = np.argsort(xbuff)
xbuff = [xbuff[i] for i in ssp]
ybuff = [ybuff[i] for i in ssp]
# fit buffer with low-order poly
buffit = np.polyfit(xbuff, ybuff, 3)
# plot buffer fit
if self.config.instrument.plot_level >= 1:
p = figure(title=self.action.args.plotlabel + ' Buffer Region',
x_axis_label='Slice Pos (px)',
y_axis_label='Ratio',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xbuff, ybuff)
p.line(xbuff, np.polyval(buffit, xbuff), line_color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# get all buffer points in image
qbuf = [i for i, v in enumerate(posmap.data.flat)
if (xinter-buffer) <= v <= (xinter+buffer)]
# apply buffer correction to all buffer points in newflat
for i in qbuf:
newflat.flat[i] = \
(resflat[1] + resflat[0] * posmap.data.flat[i]) / \
np.polyval(buffit, posmap.data.flat[i]) * newflat.flat[i]
self.logger.info("Vignetting correction complete.")
self.logger.info("Fitting master illumination")
# now fit master flat
# get reference slice points
qref = [i for i in q if ffleft <= posmap.data.flat[i] <= ffright]
xfr = wavemap.data.flat[qref]
yfr = newflat.flat[qref]
# sort on wavelength
s = np.argsort(xfr)
xfr = xfr[s]
yfr = yfr[s]
wavegood0 = wavemap.header['WAVGOOD0']
wavegood1 = wavemap.header['WAVGOOD1']
# correction for BM where we see a ledge
if 'BM' in self.action.args.grating:
ledge_wave = bm_ledge_position(self.action.args.cwave)
self.logger.info("BM ledge calculated wavelength "
"for ref slice = %.2f (A)" % ledge_wave)
if wavegood0 <= ledge_wave <= wavegood1:
self.logger.info("BM grating requires correction")
qledge = [i for i, v in enumerate(xfr)
if ledge_wave-25 <= v <= ledge_wave+25]
xledge = [xfr[i] for i in qledge]
yledge = [yfr[i] for i in qledge]
s = np.argsort(xledge)
xledge = [xledge[i] for i in s]
yledge = [yledge[i] for i in s]
win = boxcar(250)
smyledge = sp.signal.convolve(yledge,
win, mode='same') / sum(win)
ylmax = np.max(yledge)
ylmin = np.min(yledge)
fpoints = np.arange(0, 100) / 100. * 50 + (ledge_wave-25)
ledgefit, ledgemsk = Bspline.iterfit(np.asarray(xledge),
smyledge, fullbkpt=fpoints,
upper=1, lower=1)
ylfit, _ = ledgefit.value(np.asarray(fpoints))
deriv = -(np.roll(ylfit, 1) - np.roll(ylfit, -1)) / 2.0
# trim edges
trm = int(len(deriv)/5)
deriv = deriv[trm:-trm]
xvals = fpoints[trm:-trm]
peaks, _ = find_peaks(deriv, height=100)
if len(peaks) != 1:
self.logger.warning("Extra peak found!")
p = figure(title=self.action.args.plotlabel +
' Ledge', x_axis_label='Wavelength (A)',
y_axis_label='Value',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xledge, smyledge, fill_color='green')
p.line(fpoints, ylfit)
bokeh_plot(p, self.context.bokeh_session)
input("Next? <cr>: ")
p = figure(title=self.action.args.plotlabel +
' Deriv', x_axis_label='px',
y_axis_label='Value',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
xx = list(range(len(deriv)))
ylim = get_plot_lims(deriv)
p.circle(xx, deriv)
for pk in peaks:
p.line([pk, pk], ylim)
bokeh_plot(p, self.context.bokeh_session)
print("Please indicate the integer pixel value of the peak")
ipk = int(input("Peak? <int>: "))
else:
ipk = peaks[0]
apk = xvals[ipk]
if self.config.instrument.plot_level >= 3:
p = figure(
title=self.action.args.plotlabel + ' Peak of ledge',
x_axis_label='Wave (A)',
y_axis_label='Value',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xvals, deriv, legend_label='Data')
p.line([apk, apk], [-50, 200], line_color='red',
legend_label='Pk')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
xlow = apk - 3 - 5
xhi = apk - 3
zlow = apk + 3
zhi = apk + 3 + 5
qlow = [i for i, v in enumerate(fpoints) if xlow <= v <= xhi]
xlf = np.asarray([fpoints[i] for i in qlow])
ylf = np.asarray([ylfit[i] for i in qlow])
lowfit = np.polyfit(xlf, ylf, 1)
qhi = [i for i, v in enumerate(fpoints) if zlow <= v <= zhi]
xlf = np.asarray([fpoints[i] for i in qhi])
ylf = np.asarray([ylfit[i] for i in qhi])
hifit = np.polyfit(xlf, ylf, 1)
ratio = (hifit[1] + hifit[0] * apk) / \
(lowfit[1] + lowfit[0] * apk)
self.logger.info("BM ledge ratio: %.3f" % ratio)
# correct flat data
qcorr = [i for i, v in enumerate(xfr) if v >= apk]
for i in qcorr:
yfr[i] /= ratio
# plot BM ledge
if self.config.instrument.plot_level >= 1:
p = figure(
title=self.action.args.plotlabel + ' BM Ledge Region',
x_axis_label='Wave (A)',
y_axis_label='Value',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
# Input data
p.circle(xledge, yledge, fill_color='blue',
legend_label='Data')
# correct input data
qcorrect = [i for i, v in enumerate(xledge) if v >= apk]
xplt = []
yplt = []
for i in qcorrect:
xplt.append(xledge[i])
yplt.append(yledge[i] / ratio)
p.circle(xplt, yplt, fill_color='orange',
legend_label='Corrected')
p.line(fpoints, ylfit, line_color='red', legend_label='Fit')
p.line([xlow, xlow], [ylmin, ylmax], line_color='blue')
p.line([xhi, xhi], [ylmin, ylmax], line_color='blue')
p.line([zlow, zlow], [ylmin, ylmax], line_color='black')
p.line([zhi, zhi], [ylmin, ylmax], line_color='black')
p.line(fpoints, lowfit[1] + lowfit[0] * fpoints,
line_color='purple')
p.line(fpoints, hifit[1] + hifit[0] * fpoints,
line_color='green')
p.line([apk, apk], [ylmin, ylmax], line_color='green',
legend_label='Pk')
p.y_range = Range1d(ylmin, ylmax)
p.legend.location = 'top_left'
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# END: handling BM grating ledge
# if we are fitting a twilight flat, treat it like a sky image with a
# larger number of knots
if twiflat:
knots = int(ny * knotspp)
else:
knots = 100
self.logger.info("Using %d knots for bspline fit" % knots)
# generate a fit from ref slice points
bkpt = np.min(xfr) + np.arange(knots+1) * \
(np.max(xfr) - np.min(xfr)) / knots
sftr, _ = Bspline.iterfit(xfr[nrefx:-nrefx], yfr[nrefx:-nrefx],
fullbkpt=bkpt)
yfitr, _ = sftr.value(xfr)
# generate a blue slice spectrum bspline fit
blueslice = 12
blueleft = 60 / xbin
blueright = 80 / xbin
qb = [i for i, v in enumerate(slicemap.data.flat) if v == blueslice]
qblue = [i for i in qb if blueleft <= posmap.data.flat[i] <= blueright]
xfb = wavemap.data.flat[qblue]
yfb = newflat.flat[qblue]
s = np.argsort(xfb)
xfb = xfb[s]
yfb = yfb[s]
bkpt = np.min(xfb) + np.arange(knots+1) * \
(np.max(xfb) - np.min(xfb)) / knots
sftb, _ = Bspline.iterfit(xfb[nrefx:-nrefx], yfb[nrefx:-nrefx],
fullbkpt=bkpt)
yfitb, _ = sftb.value(xfb)
# generate a red slice spectrum bspline fit
redslice = 23
redleft = 60 / xbin
redright = 80 / xbin
qr = [i for i, v in enumerate(slicemap.data.flat) if v == redslice]
qred = [i for i in qr if redleft <= posmap.data.flat[i] <= redright]
xfd = wavemap.data.flat[qred]
yfd = newflat.flat[qred]
s = np.argsort(xfd)
xfd = xfd[s]
yfd = yfd[s]
bkpt = np.min(xfd) + np.arange(knots + 1) * \
(np.max(xfd) - np.min(xfd)) / knots
sftd, _ = Bspline.iterfit(xfd[nrefx:-nrefx], yfd[nrefx:-nrefx],
fullbkpt=bkpt)
yfitd, _ = sftd.value(xfd)
# waves
minwave = np.min(xfb)
maxwave = np.max(xfd)
# are we a twilight flat?
if twiflat:
nwaves = int(ny * knotspp)
else:
nwaves = 1000
waves = minwave + (maxwave - minwave) * np.arange(nwaves+1) / nwaves
if self.config.instrument.plot_level >= 1:
# output filename stub
rbfnam = "redblue_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
if xbin == 1:
stride = int(len(xfr) / 8000.)
if stride <= 0:
stride = 1
else:
stride = 1
xrplt = xfr[::stride]
yrplt = yfitr[::stride]
yrplt_d = yfr[::stride]
xbplt = xfb[::stride]
ybplt = yfitb[::stride]
ybplt_d = yfb[::stride]
xdplt = xfd[::stride]
ydplt = yfitd[::stride]
ydplt_d = yfd[::stride]
p = figure(
title=self.action.args.plotlabel + ' Blue/Red fits',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(xrplt, yrplt, line_color='black', legend_label='Ref')
p.circle(xrplt, yrplt_d, size=1, line_alpha=0., fill_color='black')
p.line(xbplt, ybplt, line_color='blue', legend_label='Blue')
p.circle(xbplt, ybplt_d, size=1, line_alpha=0., fill_color='blue')
p.line(xdplt, ydplt, line_color='red', legend_label='Red')
p.circle(xdplt, ydplt_d, size=1, line_alpha=0., fill_color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=rbfnam+'.png')
wavebuffer = 0.1
minrwave = np.min(xfr)
maxrwave = np.max(xfr)
wavebuffer2 = 0.05
wlb0 = minrwave+(maxrwave-minrwave)*wavebuffer2
wlb1 = minrwave+(maxrwave-minrwave)*wavebuffer
wlr0 = minrwave+(maxrwave-minrwave)*(1.-wavebuffer)
wlr1 = minrwave+(maxrwave-minrwave)*(1.-wavebuffer2)
qbluefit = [i for i, v in enumerate(waves) if wlb0 < v < wlb1]
qredfit = [i for i, v in enumerate(waves) if wlr0 < v < wlr1]
nqb = len(qbluefit)
nqr = len(qredfit)
self.logger.info("Wavelength regions: blue = %.1f - %.1f, "
"red = %.1f - %.1f" % (wlb0, wlb1, wlr0, wlr1))
self.logger.info("Fit points: blue = %d, red = %d" % (nqb, nqr))
if nqb > 0:
bluefit, _ = sftb.value(waves[qbluefit])
refbluefit, _ = sftr.value(waves[qbluefit])
blue_zero_cross = np.nanmin(bluefit) <= 0. or np.nanmin(
refbluefit) <= 0.
bluelinfit = np.polyfit(waves[qbluefit], refbluefit/bluefit, 1)
bluelinfity = bluelinfit[1] + bluelinfit[0] * waves[qbluefit]
else:
bluefit = None
blue_zero_cross = False
refbluefit = None
bluelinfit = None
bluelinfity = None
if nqr > 0:
redfit, _ = sftd.value(waves[qredfit])
refredfit, _ = sftr.value(waves[qredfit])
red_zero_cross = np.nanmin(redfit) <= 0. or np.nanmin(
refredfit) <= 0.
redlinfit = np.polyfit(waves[qredfit], refredfit/redfit, 1)
redlinfity = redlinfit[1] + redlinfit[0] * waves[qredfit]
else:
redfit = None
red_zero_cross = False
refredfit = None
redlinfit = None
redlinfity = None
if blue_zero_cross:
self.logger.info("Blue extension zero crossing detected")
if red_zero_cross:
self.logger.info("Red extension zero crossing detected")
if self.config.instrument.plot_level >= 1:
if nqb > 1:
# plot blue fits
p = figure(
title=self.action.args.plotlabel + ' Blue fits',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(waves[qbluefit], refbluefit, line_color='black',
legend_label='Ref')
p.circle(waves[qbluefit], refbluefit, fill_color='black')
p.line(waves[qbluefit], bluefit, line_color='blue',
legend_label='Blue')
p.circle(waves[qbluefit], bluefit, fill_color='blue')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# plot blue ratios
p = figure(
title=self.action.args.plotlabel + ' Blue ratios',
x_axis_label='Wave (A)',
y_axis_label='Ratio',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(waves[qbluefit], refbluefit/bluefit, line_color='black',
legend_label='Ref')
p.circle(waves[qbluefit], refbluefit/bluefit,
fill_color='black')
p.line(waves[qbluefit], bluelinfity,
line_color='blue', legend_label='Blue')
p.circle(waves[qbluefit], bluelinfity,
fill_color='blue')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
if nqr > 1:
# plot red fits
p = figure(
title=self.action.args.plotlabel + ' Red fits',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(waves[qredfit], refredfit, line_color='black',
legend_label='Ref')
p.circle(waves[qredfit], refredfit, fill_color='black')
p.line(waves[qredfit], redfit, line_color='red',
legend_label='Red')
p.circle(waves[qredfit], redfit, fill_color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# plot red ratios
p = figure(
title=self.action.args.plotlabel + ' Red ratios',
x_axis_label='Wave (A)',
y_axis_label='Ratio',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(waves[qredfit], refredfit/redfit, line_color='black',
legend_label='Ref')
p.circle(waves[qredfit], refredfit/redfit, fill_color='black')
p.line(waves[qredfit], redlinfity,
line_color='red', legend_label='Red')
p.circle(waves[qredfit], redlinfity, fill_color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# at this point we are going to try to merge the points
self.logger.info("Correcting points outside %.1f - %.1f A"
% (minrwave, maxrwave))
qselblue = [i for i, v in enumerate(xfb) if v <= minrwave]
qselred = [i for i, v in enumerate(xfd) if v >= maxrwave]
nqsb = len(qselblue)
nqsr = len(qselred)
blue_all_tie = yfitr[0]
red_all_tie = yfitr[-1]
self.logger.info("Blue/Red ref tie values: %.3f, %.3f"
% (blue_all_tie, red_all_tie))
if nqsb > 0:
self.logger.info("Blue ext tie value: %.3f" % yfb[qselblue[-1]])
if blue_zero_cross:
blue_offset = yfb[qselblue[-1]] - blue_all_tie
bluefluxes = [yfb[i] - blue_offset for i in qselblue]
self.logger.info("Blue zero crossing, only applying offset")
else:
blue_offset = yfb[qselblue[-1]] * \
(bluelinfit[1]+bluelinfit[0]*xfb[qselblue[-1]]) \
- blue_all_tie
bluefluxes = [yfb[i] * (bluelinfit[1]+bluelinfit[0]*xfb[i])
- blue_offset for i in qselblue]
self.logger.info("Blue linear ratio fit scaling applied")
self.logger.info("Blue offset of %.2f applied" % blue_offset)
else:
bluefluxes = None
if nqsr > 0:
self.logger.info("Red ext tie value: %.3f" % yfd[qselred[0]])
if red_zero_cross:
red_offset = yfd[qselred[0]] - red_all_tie
redfluxes = [yfd[i] - red_offset for i in qselred]
self.logger.info("Red zero crossing, only applying offset")
else:
red_offset = yfd[qselred[0]] * \
(redlinfit[1]+redlinfit[0]*xfd[qselred[0]]) \
- red_all_tie
redfluxes = [yfd[i] * (redlinfit[1]+redlinfit[0]*xfd[i])
- red_offset for i in qselred]
self.logger.info("Red linear ratio fit scaling applied")
self.logger.info("Red offset of %.2f applied" % red_offset)
else:
redfluxes = None
allx = xfr
allfx = xfr[nrefx:-nrefx]
ally = yfr[nrefx:-nrefx]
if nqsb > 0:
bluex = xfb[qselblue]
allx = np.append(bluex, allx)
allfx = np.append(bluex[nrefx:], allfx)
ally = np.append(bluefluxes[nrefx:], ally)
if nqsr > 0:
redx = xfd[qselred]
allx = np.append(allx, redx)
allfx = np.append(allfx, redx[:-nrefx])
ally = np.append(ally, redfluxes[:-nrefx])
s = np.argsort(allx)
allx = allx[s]
s = np.argsort(allfx)
allfx = allfx[s]
ally = ally[s]
bkpt = np.min(allx) + np.arange(knots+1) * \
(np.max(allx) - np.min(allx)) / knots
sftall, _ = Bspline.iterfit(allfx, ally, fullbkpt=bkpt)
yfitall, _ = sftall.value(allx)
if self.config.instrument.plot_level >= 1:
# output filename stub
fltfnam = "flat_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
if xbin == 1:
stride = int(len(allx) / 8000.)
else:
stride = 1
xplt = allfx[::stride]
yplt = ally[::stride]
fxplt = allx[::stride]
fplt = yfitall[::stride]
yran = [np.nanmin(ally), np.nanmax(ally)]
p = figure(
title=self.action.args.plotlabel + ' Master Illumination',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xplt, yplt, size=1, line_alpha=0., fill_color='black',
legend_label='Data')
p.line(fxplt, fplt, line_color='red', legend_label='Fit',
line_width=2)
p.line([minrwave, minrwave], yran, line_color='orange',
legend_label='Cor lim')
p.line([maxrwave, maxrwave], yran, line_color='orange')
p.legend.location = "top_left"
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=fltfnam+".png")
# OK, Now we have extended to the full range... so... we are going to
# make a ratio flat!
comflat = np.zeros(newflat.shape, dtype=float)
qz = [i for i, v in enumerate(wavemap.data.flat) if v >= 0]
comvals = sftall.value(wavemap.data.flat[qz])
comflat.flat[qz] = comvals
ratio = np.zeros(newflat.shape, dtype=float)
qzer = [i for i, v in enumerate(newflat.flat) if v != 0]
ratio.flat[qzer] = comflat.flat[qzer] / newflat.flat[qzer]
# trim negative points
qq = [i for i, v in enumerate(ratio.flat) if v < 0]
if len(qq) > 0:
ratio.flat[qq] = 0.0
# trim the high points near edges of slice
qq = [i for i, v in enumerate(ratio.flat) if v >= 3. and
(posmap.data.flat[i] <= 4/xbin or
posmap.data.flat[i] >= 136/xbin)]
if len(qq) > 0:
ratio.flat[qq] = 0.0
# don't correct low signal points
qq = [i for i, v in enumerate(newflat.flat) if v < 30.]
if len(qq) > 0:
ratio.flat[qq] = 1.0
# get master flat output name
mfname = stack_list[0].split('.fits')[0] + '_' + suffix + '.fits'
log_string = MakeMasterFlat.__module__
stacked.header['IMTYPE'] = self.action.args.new_type
stacked.header['HISTORY'] = log_string
stacked.header['MASTFLAT'] = (True, 'master flat image?')
stacked.header['WAVMAPF'] = wmf
stacked.header['SLIMAPF'] = slf
stacked.header['POSMAPF'] = pof
# store flat in output frame
stacked.data = ratio
# output master flat
kcwi_fits_writer(stacked, output_file=mfname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=stacked, suffix=suffix,
newtype=self.action.args.new_type,
filename=self.action.args.name)
self.context.proctab.write_proctab()
self.logger.info(log_string)
return self.action.args
def _perform(self):
# Header keyword to update
key = 'SCATSUB'
keycom = "was scattered light subtracted?"
# Skip if nod-and-shuffle
if self.action.args.nasmask:
self.logger.info("NAS Mask: skipping scattered light subtraction")
self.action.args.ccddata.header[key] = (False, keycom)
elif self.config.instrument.skipscat:
self.logger.info("Skipping scattered light subtraction by request")
self.action.args.ccddata.header[key] = (False, keycom)
else:
# Binning
ybin = self.action.args.ybinsize
# Get size of image
siz = self.action.args.ccddata.data.shape
# Get x range for scattered light
x0 = int(siz[1] / 2 - 180 / self.action.args.xbinsize)
x1 = int(siz[1] / 2 + 180 / self.action.args.xbinsize)
# Get y limits
y0 = 0
# y1 = int(siz[0] / 2 - 1)
# y2 = y1 + 1
y3 = siz[0]
# print("x limits: %d, %d, y limits: %d, %d" % (x0, x1, y0, y3))
# Y data values
yvals = np.nanmedian(self.action.args.ccddata.data[y0:y3, x0:x1],
axis=1)
# Fix extreme values
yvals[0] = np.nanmedian(yvals[1:10])
yvals[-1] = np.nanmedian(yvals[-11:-2])
# X data values
xvals = np.arange(len(yvals), dtype=np.float)
# filter window
fwin = 151
scat = savgol_filter(yvals, fwin, 3)
signal_to_noise = np.mean(scat) / np.nanstd(yvals - scat)
if signal_to_noise < 25.:
if signal_to_noise < 5:
fwin = 501
else:
fwin = 303
scat = savgol_filter(yvals, fwin, 3)
signal_to_noise = np.mean(scat) / np.nanstd(yvals - scat)
self.logger.info("Smoothing scattered light with window of %d px" %
fwin)
self.logger.info("Mean signal to noise = %.2f" % signal_to_noise)
if self.config.instrument.plot_level >= 1:
# output filename stub
scfnam = "scat_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
# plot
p = figure(title=self.action.args.plotlabel +
"SCATTERED LIGHT FIT",
x_axis_label='y pixel',
y_axis_label='e-',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xvals, yvals, legend_label="Scat")
p.line(xvals,
scat,
color='red',
line_width=3,
legend_label="Smooth")
p.legend.location = "top_left"
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=scfnam + ".png")
# Subtract scattered light
self.logger.info("Starting scattered light subtraction")
for ix in range(0, siz[1]):
self.action.args.ccddata.data[y0:y3, ix] = \
self.action.args.ccddata.data[y0:y3, ix] - scat
self.action.args.ccddata.header[key] = (True, keycom)
log_string = SubtractScatteredLight.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
# write out int image
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="intd")
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix="intd")
self.context.proctab.write_proctab()
return self.action.args
