def traceFootprints(self, rc):
"""
This primitive will create and append a 'TRACEFP' Bintable HDU to the
AD object. The content of this HDU is the footprints information
from the espectroscopic flat in the SCI array.
:param logLevel: Verbosity setting for log messages to the screen.
:type logLevel: integer from 0-6, 0=nothing to screen, 6=everything to
screen. OR the message level as a string (i.e.,
'critical', 'status', 'fullinfo'...)
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "", "starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Check whether this primitive has been run previously
if ad.phu_get_key_value("TRACEFP"):
log.warning("%s has already been processed by traceSlits" \
% (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Call the user level function
try:
adout = trace_footprints(ad,
function=rc["function"],
order=rc["order"],
trace_threshold=rc["trace_threshold"])
except:
log.warning("Error in traceFootprints with file: %s" %
ad.filename)
# Change the filename
adout.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects.
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def cutFootprints(self, rc):
"""
This primitive will create and append multiple HDU to the output
AD object. Each HDU correspond to a rectangular cut containing a
slit from a MOS Flat exposure or a XD flat exposure as in the
Gnirs case.
:param logLevel: Verbosity setting for log messages to the screen.
:type logLevel: integer from 0-6, 0=nothing to screen, 6=everything to
screen. OR the message level as a string (i.e.,
'critical', 'status', 'fullinfo'...)
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "cutFootprints", "starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Call the user level function
# Check that the input ad has the TRACEFP extension,
# otherwise, create it.
if ad['TRACEFP'] == None:
ad = trace_footprints(ad)
log.stdinfo("Cutting_footprints for: %s" % ad.filename)
try:
adout = cut_footprints(ad)
except:
log.error("Error in cut_slits with file: %s" % ad.filename)
# DO NOT add this input ad to the adoutput_lis
continue
# Change the filename
adout.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects.
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def rejectCosmicRays(self, rc):
"""
Reject cosmic rays using a custom implementation of the LACosmic
algorithm
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "rejectCosmicRays",
"starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["rejectCosmicRays"]
# Initialise a list of output AstroData objects
adoutput_list = []
# Define the Laplacian and growth kernels for L.A.Cosmic
laplace_kernel = np.array([
[0.0, -1.0, 0.0],
[-1.0, 4.0, -1.0],
[0.0, -1.0, 0.0],
])
growth_kernel = np.ones((3, 3), dtype=np.float64)
# Loop over the input AstroData objects
for ad in rc.get_inputs_as_astrodata():
# Check if the rejectCosmicRays primitive has been run previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has "
"already been processed by rejectCosmicRays"
% ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Define an array that will hold the cosmic ray flagging
# Note that we're deliberately not using the BPM at this stage,
# otherwise the algorithm will start searching for cosmic rays
# around pixels that have been flagged bad for another reason.
cosmic_bpm = np.zeros_like(ad["SCI", 1].data, dtype=bool)
new_crs = 1
# Start with a fresh copy of the data
clean_data = np.copy(ad["SCI", 1].data)
# Define the function for performing the median-replace of cosmic
# ray pixels
# Note that this is different from a straight median filter, as we
# *don't* want to include the central pixel
fp = [[1, 1, 1],
[1, 0, 1],
[1, 1, 1]]
median_replace = functools.partial(scipy.ndimage.generic_filter,
function=np.median, footprint=fp)
log.stdinfo('Doing CR removal for %s' % ad.filename)
no_passes = 0
while new_crs > 0 and no_passes < 1:
no_passes += 1
curr_crs = np.count_nonzero(cosmic_bpm)
# Median out the pixels already defined as cosmic rays
log.stdinfo('Pass %d: Wiping over previously found bad pix' %
no_passes)
if curr_crs > 0:
clean_data[cosmic_bpm] = median_replace(
clean_data)[cosmic_bpm]
# Actually do the cosmic ray subtraction here
# ------
# STEP 1
# Construct a model for sky lines to subtract
# TODO: Add option for 'wave' keyword, which parametrizes
# an input wavelength solution function
# ------
log.stdinfo('Pass %d: Building sky model' % no_passes)
sky_model = scipy.ndimage.median_filter(clean_data, size=[7, 1])
m5_model = scipy.ndimage.median_filter(clean_data, size=[5, 5])
subbed_data = clean_data - sky_model
# ------
# STEP 2
# Remove object spectra
# TODO: Determine if this is necessary - PyWiFeS does not use it
# ------
# ------
# STEP 3
# Compute 2nd-order Laplacian of input frame
# This is 'curly L' in van Dokkum 2001
# ------
# Subsample the data
log.stdinfo('Pass %d: Computing Laplacian' % no_passes)
subsampling = rc["subsampling"]
data_shape = ad["SCI", 1].data.shape
subsampl_data = np.repeat(np.repeat(
ad["SCI", 1].data, subsampling, axis=1),
subsampling, axis=0
)
# Convolve the subsampled data with the Laplacian kernel,
# trimming off the edges this introduces
# Bring any negative values up to 0
init_conv_data = scipy.signal.convolve2d(
subsampl_data, laplace_kernel)[1:-1, 1:-1]
init_conv_data[np.nonzero(init_conv_data <= 0.)] = 0.
# Reverse the subsampling, returning the
# correctly-convolved image
conv_data = np.reshape(init_conv_data,
(
data_shape[0],
init_conv_data.shape[0] //
data_shape[0],
data_shape[1],
init_conv_data.shape[1] //
data_shape[1],
)).mean(axis=3).mean(axis=1)
# ------
# STEP 4
# Construct noise model, and use it to generate the
# 'sigma_map' S
# This is the equivalent of equation (11) of van Dokkum 2001
# ------
log.stdinfo('Pass %d: Constructing sigma map' % no_passes)
gain = ad.gain()
read_noise = ad.read_noise()
noise = (1.0 / gain) * ((gain * m5_model + read_noise**2)**0.5)
noise_min = 0.00001
noise[np.nonzero(noise <= noise_min)] = noise_min
# div by 2 to correct convolution counting
# FIXME: Why is it divided by *two* times the noise?
sigmap = conv_data / (2.0 * noise)
# Remove large structure with a 5x5 median filter
# Equation (13) of van Dokkum 2001, generates S'
sig_smooth = scipy.ndimage.median_filter(sigmap, size=[5, 5])
sig_detrend = sigmap - sig_smooth
# ------
# STEP 5
# Identify the potential cosmic rays
# ------
log.stdinfo('Pass %d: Flagging cosmic rays' % no_passes)
# Construct the fine-structure image (F, eqn 14 of van Dokkum)
m3 = scipy.ndimage.median_filter(subbed_data, size=[3, 3])
fine_struct = m3 - scipy.ndimage.median_filter(m3, size=[7, 7])
# Pixels are flagged as being cosmic rays if:
# - The sig_detrend image (S') is > sigma_lim
# - The contrast between the Laplacian image (L+) and the
# fine-structure image (F) is greater than f_lim
sigma_lim = rc['sigma_lim']
f_lim = rc['f_lim']
cosmic_bpm[np.logical_and(sig_detrend > sigma_lim,
(conv_data/fine_struct) > f_lim)] = 1
new_crs = np.count_nonzero(cosmic_bpm) - curr_crs
log.stdinfo('Found %d CR pixels in pass %d' % (new_crs,
no_passes, ))
# TODO: Determine whether to alter pix or alter BPM
# For the moment, go with Mike Ireland's suggestion to require
# a BPM update
curr_bpm = ad["DQ", 1].data
curr_bpm[cosmic_bpm] = True
ad['DQ', 1].data = curr_bpm
# Append the ad to the output list
adoutput_list.append(ad)
# Finish
rc.report_output(adoutput_list)
yield rc
def standardizeGeminiHeaders(self, rc):
"""
This primitive is used to make the changes and additions to the
keywords in the headers of Gemini data.
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(
gt.log_message("primitive", "standardizeGeminiHeaders",
"starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["standardizeGeminiHeaders"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Check whether the standardizeGeminiHeaders primitive has been run
# previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has "
"already been processed by "
"standardizeGeminiHeaders" % ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Standardize the headers of the input AstroData object. Update the
# keywords in the headers that are common to all Gemini data.
log.status("Updating keywords that are common to all Gemini data")
# Original name
ad.store_original_name()
# Number of science extensions
gt.update_key(adinput=ad,
keyword="NSCIEXT",
value=ad.count_exts("SCI"),
comment=None,
extname="PHU",
keyword_comments=self.keyword_comments)
# Number of extensions
gt.update_key(adinput=ad,
keyword="NEXTEND",
value=len(ad),
comment=None,
extname="PHU",
keyword_comments=self.keyword_comments)
# Physical units (assuming raw data has units of ADU)
gt.update_key(adinput=ad,
keyword="BUNIT",
value="adu",
comment=None,
extname="SCI",
keyword_comments=self.keyword_comments)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad,
primname=self.myself(),
keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
def attachWavelengthSolution(self, rc):
# Instantiate the log
log = logutils.get_logger(__name__)
# Define the keyword to be used for the time stamp
timestamp_key = self.timestamp_keys["attachWavelengthSolution"]
# Log the standard "starting primitive" debug message
log.debug(
gt.log_message("primitive", "attachWavelengthSolution",
"starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Check for a user-supplied arc
adinput = rc.get_inputs_as_astrodata()
arc_param = rc["arc"]
arc_dict = None
if arc_param is not None:
# The user supplied an input to the arc parameter
if not isinstance(arc_param, list):
arc_list = [arc_param]
else:
arc_list = arc_param
# Convert filenames to AD instances if necessary
tmp_list = []
for arc in arc_list:
if type(arc) is not AstroData:
arc = AstroData(arc)
tmp_list.append(arc)
arc_list = tmp_list
arc_dict = gt.make_dict(key_list=adinput, value_list=arc_list)
for ad in adinput:
if arc_dict is not None:
arc = arc_dict[ad]
else:
arc = rc.get_cal(ad, "processed_arc")
# Take care of the case where there was no arc
if arc is None:
log.warning("Could not find an appropriate arc for %s" \
% (ad.filename))
adoutput_list.append(ad)
continue
else:
arc = AstroData(arc)
wavecal = arc["WAVECAL"]
if wavecal is not None:
# Remove old versions
if ad["WAVECAL"] is not None:
for wc in ad["WAVECAL"]:
ad.remove((wc.extname(), wc.extver()))
# Append new solution
ad.append(wavecal)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad,
primname=self.myself(),
keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
adoutput_list.append(ad)
else:
log.warning("No wavelength solution found for %s" %
ad.filename)
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def mosaicADdetectors(self, rc): # Uses python MosaicAD script
"""
This primitive will mosaic the SCI frames of the input images, along
with the VAR and DQ frames if they exist.
:param tile: tile images instead of mosaic
:type tile: Python boolean (True/False), default is False
"""
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "mosaicADdetectors", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["mosaicADdetectors"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Validate Data
#if (ad.phu_get_key_value('GPREPARE')==None) and \
# (ad.phu_get_key_value('PREPARE')==None):
# raise Errors.InputError("%s must be prepared" % ad.filename)
# Check whether the mosaicDetectors primitive has been run
# previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has " \
"already been processed by mosaicDetectors" \
% (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# If the input AstroData object only has one extension, there is no
# need to mosaic the detectors
if ad.count_exts("SCI") == 1:
log.stdinfo("No changes will be made to %s, since it " \
"contains only one extension" % (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Get the necessary parameters from the RC
tile = rc["tile"]
log.stdinfo("Mosaicking %s ..." % ad.filename)
log.stdinfo("MosaicAD: Using tile: %s ..." % tile)
#t1 = time.time()
mo = MosaicAD(ad,
mosaic_ad_function=gemini_mosaic_function,
dq_planes=rc['dq_planes'])
adout = mo.as_astrodata(tile=tile)
#t2 = time.time()
#print '%s took %0.3f ms' % ('as_astrodata', (t2-t1)*1000.0)
# Verify mosaicAD was actually run on the file
# then log file names of successfully reduced files
if adout.phu_get_key_value("MOSAIC"):
log.fullinfo("File "+adout.filename+\
" was successfully mosaicked")
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=adout,
primname=self.myself(),
keyword=timestamp_key)
# Change the filename
adout.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc