def test_inputs_match(self, astrofaker):
ad1 = astrofaker.create("GMOS-S", mode="IMAGE")
ad1.init_default_extensions()
ad2 = astrofaker.create("GMOS-S", mode="IMAGE")
ad2.init_default_extensions()
gt.check_inputs_match(ad1, ad2)
def test_inputs_match_different_shapes(self, astrofaker):
ad1 = astrofaker.create("GMOS-S", mode="IMAGE")
ad1.init_default_extensions()
for ext in ad1:
ext.data = ext.data[20:-20, 20:-20]
ad2 = astrofaker.create("GMOS-S", mode="IMAGE")
ad2.init_default_extensions()
with pytest.raises(ValueError):
gt.check_inputs_match(ad1, ad2)
gt.check_inputs_match(ad1, ad2, check_shape=False)
def biasCorrect(self, adinputs=None, suffix=None, bias=None, do_bias=True):
"""
The biasCorrect primitive will subtract the science extension of the
input bias frames from the science extension of the input science
frames. The variance and data quality extension will be updated, if
they exist. If no bias is provided, getProcessedBias will be called
to ensure a bias exists for every adinput.
Parameters
----------
suffix: str
suffix to be added to output files
bias: str/list of str
bias(es) to subtract
do_bias: bool
perform bias subtraction?
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
if not do_bias:
log.warning("Bias correction has been turned off.")
return adinputs
if bias is None:
self.getProcessedBias(adinputs, refresh=False)
bias_list = self._get_cal(adinputs, 'processed_bias')
else:
bias_list = bias
# Provide a bias AD object for every science frame
for ad, bias in zip(
*gt.make_lists(adinputs, bias_list, force_ad=True)):
if ad.phu.get(timestamp_key):
log.warning("No changes will be made to {}, since it has "
"already been processed by biasCorrect".format(
ad.filename))
continue
if bias is None:
if 'qa' in self.mode:
log.warning("No changes will be made to {}, since no "
"bias was specified".format(ad.filename))
continue
else:
raise OSError('No processed bias listed for {}'.format(
ad.filename))
try:
gt.check_inputs_match(ad,
bias,
check_filter=False,
check_units=True)
except ValueError:
bias = gt.clip_auxiliary_data(ad, aux=bias, aux_type='cal')
# An Error will be raised if they don't match now
gt.check_inputs_match(ad,
bias,
check_filter=False,
check_units=True)
log.fullinfo('Subtracting this bias from {}:\n{}'.format(
ad.filename, bias.filename))
ad.subtract(bias)
# Record bias used, timestamp, and update filename
ad.phu.set('BIASIM', bias.filename,
self.keyword_comments['BIASIM'])
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=suffix, strip=True)
if bias.path:
add_provenance(ad, bias.filename,
md5sum(bias.path) or "", self.myself())
timestamp = datetime.now()
return adinputs
def slitIllumCorrect(self,
adinputs=None,
slit_illum=None,
do_illum=True,
suffix="_illumCorrected"):
"""
This primitive will divide each SCI extension of the inputs by those
of the corresponding slit illumination image. If the inputs contain
VAR or DQ frames, those will also be updated accordingly due to the
division on the data.
Parameters
----------
adinputs : list of AstroData
Data to be corrected.
slit_illum : str or AstroData
Slit illumination path or AstroData object.
do_illum: bool, optional
Perform slit illumination correction? (Default: True)
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
qecorr_key = self.timestamp_keys['QECorrect']
if not do_illum:
log.warning("Slit Illumination correction has been turned off.")
return adinputs
if slit_illum is None:
raise NotImplementedError
else:
slit_illum_list = slit_illum
# Provide a Slit Illum Ad object for every science frame
ad_outputs = []
for ad, slit_illum_ad in zip(
*gt.make_lists(adinputs, slit_illum_list, force_ad=True)):
if ad.phu.get(timestamp_key):
log.warning("No changes will be made to {}, since it has "
"already been processed by flatCorrect".format(
ad.filename))
continue
if slit_illum_ad is None:
if self.mode in ['sq']:
raise OSError(
"No processed slit illumination listed for {}".format(
ad.filename))
else:
log.warning("No changes will be made to {}, since no slit "
"illumination has been specified".format(
ad.filename))
continue
gt.check_inputs_match(ad, slit_illum_ad, check_shape=False)
if not all(
[e1.shape == e2.shape for (e1, e2) in zip(ad, slit_illum_ad)]):
slit_illum_ad = gt.clip_auxiliary_data(adinput=[ad],
aux=[slit_illum_ad])[0]
log.info("Dividing the input AstroData object {} by this \n"
"slit illumination file: \n{}".format(
ad.filename, slit_illum_ad.filename))
ad_out = deepcopy(ad)
ad_out.divide(slit_illum_ad)
# Update the header and filename, copying QECORR keyword from flat
ad_out.phu.set("SLTILLIM", slit_illum_ad.filename,
self.keyword_comments["SLTILLIM"])
try:
qecorr_value = slit_illum_ad.phu[qecorr_key]
except KeyError:
pass
else:
log.fullinfo(
"Copying {} keyword from slit illumination".format(
qecorr_key))
ad_out.phu.set(qecorr_key, qecorr_value,
slit_illum_ad.phu.comments[qecorr_key])
gt.mark_history(ad_out,
primname=self.myself(),
keyword=timestamp_key)
ad_out.update_filename(suffix=suffix, strip=True)
if slit_illum_ad.path:
add_provenance(ad_out, slit_illum_ad.filename,
md5sum(slit_illum_ad.path) or "", self.myself())
ad_outputs.append(ad_out)
return ad_outputs
def fringeCorrect(self, adinputs=None, **params):
"""
Correct science frames for the effects of fringing, using a fringe
frame. The fringe frame is obtained either from a specified parameter,
or the "fringe" stream, or the calibration database. This is basically
a bookkeeping wrapper for subtractFringe(), which does all the work.
Parameters
----------
suffix: str
suffix to be added to output files
fringe: list/str/AstroData/None
fringe frame(s) to subtract
do_fringe: bool/None
apply fringe correction? (None => use pipeline default for data)
scale: bool/None
scale fringe frame? (None => False if fringe frame has same
group_id() as data
scale_factor: float/sequence/None
factor(s) to scale fringe
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
fringe = params["fringe"]
scale = params["scale"]
do_cal = params["do_cal"]
# Exit now if nothing needs a correction, to avoid an error when the
# calibration search fails. If images with different exposure times
# are used, some frames may not require a correction (but the calibration
# search will succeed), so still need to check individual inputs later.
needs_correction = [self._needs_fringe_correction(ad) for ad in adinputs]
if any(needs_correction):
if do_cal == 'skip':
log.warning("Fringe correction has been turned off but is "
"recommended.")
return adinputs
else:
if do_cal == 'procmode' or do_cal == 'skip':
log.stdinfo("No input images require a fringe correction.")
return adinputs
else: # do_cal == 'force':
log.warning("Fringe correction has been forced on but may not "
"be required.")
if fringe is None:
# This logic is for QAP
try:
fringe_list = self.streams['fringe']
assert len(fringe_list) == 1
scale = False
log.stdinfo("Using fringe frame in 'fringe' stream. "
"Setting scale=False")
fringe_list = (fringe_list[0], "stream")
except (KeyError, AssertionError):
fringe_list = self.caldb.get_processed_fringe(adinputs)
else:
fringe_list = (fringe, None)
# Usual stuff to ensure that we have an iterable of the correct length
# for the scale factors regardless of what the input is
scale_factor = params["scale_factor"]
try:
factors = iter(scale_factor)
except TypeError:
factors = iter([scale_factor] * len(adinputs))
else:
# In case a single-element list was passed
if len(scale_factor) == 1:
factors = iter(scale_factor * len(adinputs))
# Get a fringe AD object for every science frame
for ad, fringe, origin, correct in zip(*gt.make_lists(
adinputs, *fringe_list, needs_correction, force_ad=(1,))):
if ad.phu.get(timestamp_key):
log.warning(f"{ad.filename}: already processed by "
"fringeCorrect. Continuing.")
continue
# Logic to deal with different exposure times where only
# some inputs might require fringe correction
# KL: for now, I'm not allowing the "force" to do anything when
# the correction is not needed.
if (do_cal == 'procmode' or do_cal == 'force') and not correct:
log.stdinfo("{} does not require a fringe correction".
format(ad.filename))
ad.update_filename(suffix=params["suffix"], strip=True)
continue
# At this point, we definitely want to do a fringe correction
# so we'd better have a fringe frame!
if fringe is None:
if 'sq' not in self.mode and do_cal != 'force':
log.warning("No changes will be made to {}, since no "
"fringe frame has been specified".
format(ad.filename))
continue
else:
log.warning(f"{ad.filename}: no fringe was specified. "
"Continuing.")
continue
# Check the inputs have matching filters, binning, and shapes
try:
gt.check_inputs_match(ad, fringe)
except ValueError:
fringe = gt.clip_auxiliary_data(adinput=ad, aux=fringe,
aux_type="cal")
gt.check_inputs_match(ad, fringe)
#
origin_str = f" (obtained from {origin})" if origin else ""
log.stdinfo(f"{ad.filename}: using the fringe frame "
f"{fringe.filename}{origin_str}")
matched_groups = (ad.group_id() == fringe.group_id())
if scale or (scale is None and not matched_groups):
factor = next(factors)
if factor is None:
factor = self._calculate_fringe_scaling(ad, fringe)
log.stdinfo("Scaling fringe frame by factor {:.3f} before "
"subtracting from {}".format(factor, ad.filename))
# Since all elements of fringe_list might be references to the
# same AD, need to make a copy before multiplying
fringe_copy = deepcopy(fringe)
fringe_copy.multiply(factor)
ad.subtract(fringe_copy)
else:
if scale is None:
log.stdinfo("Not scaling fringe frame with same group ID "
"as {}".format(ad.filename))
ad.subtract(fringe)
# Timestamp and update header and filename
ad.phu.set("FRINGEIM", fringe.filename, self.keyword_comments["FRINGEIM"])
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=params["suffix"], strip=True)
if fringe.path:
add_provenance(ad, fringe.filename, md5sum(fringe.path) or "", self.myself())
return adinputs
def test_check_inputs_match(self):
ad1 = astrodata.open(os.path.join(TESTDATAPATH, 'NIRI',
'N20130404S0372_aligned.fits'))
ad2 = astrodata.open(os.path.join(TESTDATAPATH, 'NIRI',
'N20130404S0373_aligned.fits'))
gt.check_inputs_match(ad1, ad2)
def fringeCorrect(self, adinputs=None, **params):
"""
Correct science frames for the effects of fringing, using a fringe
frame. The fringe frame is obtained either from a specified parameter,
or the "fringe" stream, or the calibration database. This is basically
a bookkeeping wrapper for subtractFringe(), which does all the work.
Parameters
----------
suffix: str
suffix to be added to output files
fringe: list/str/AstroData/None
fringe frame(s) to subtract
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
# Exit now if nothing needs a correction, to avoid an error when the
# calibration search fails. If images with different exposure times
# are used, some frames may not require a correction (but the calibration
# search will succeed), so still need to check individual inputs later.
if not any(self._needs_fringe_correction(ad) for ad in adinputs):
log.stdinfo("No input images require a fringe correction.")
return adinputs
fringe = params["fringe"]
scale = params["scale"]
if fringe is None:
try:
fringe_list = self.streams['fringe']
assert len(fringe_list) == 1
scale = False
log.stdinfo("Using fringe frame in 'fringe' stream. "
"Setting scale=False")
except (KeyError, AssertionError):
self.getProcessedFringe(adinputs)
fringe_list = self._get_cal(adinputs, "processed_fringe")
else:
fringe_list = fringe
# Usual stuff to ensure that we have an iterable of the correct length
# for the scale factors regardless of what the input is
scale_factor = params["scale_factor"]
try:
factors = iter(scale_factor)
except TypeError:
factors = iter([scale_factor] * len(adinputs))
else:
# In case a single-element list was passed
if len(scale_factor) == 1:
factors = iter(scale_factor * len(adinputs))
# Get a fringe AD object for every science frame
for ad, fringe in zip(*gt.make_lists(adinputs, fringe_list, force_ad=True)):
if ad.phu.get(timestamp_key):
log.warning("No changes will be made to {}, since it has "
"already been processed by subtractFringe".
format(ad.filename))
continue
# Check the inputs have matching filters, binning, and shapes
try:
gt.check_inputs_match(ad, fringe)
except ValueError:
fringe = gt.clip_auxiliary_data(adinput=ad, aux=fringe,
aux_type="cal")
gt.check_inputs_match(ad, fringe)
if scale:
factor = next(factors)
if factor is None:
factor = self._calculate_fringe_scaling(ad, fringe)
log.stdinfo("Scaling fringe frame by factor {:.3f} before "
"subtracting from {}".format(factor, ad.filename))
# Since all elements of fringe_list might be references to the
# same AD, need to make a copy before multiplying
fringe_copy = deepcopy(fringe)
fringe_copy.multiply(factor)
ad.subtract(fringe_copy)
else:
ad.subtract(fringe)
# Timestamp and update header and filename
ad.phu.set("FRINGEIM", fringe.filename, self.keyword_comments["FRINGEIM"])
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=params["suffix"], strip=True)
return adinputs
def processSlits(self, adinputs=None, **params):
"""
Compute and record the mean exposure epoch for a slit viewer image
The 'slit viewer image' for each observation will almost certainly
be a sequence of short exposures of the slit viewer camera,
collected together for convenience. However, it cannot be guaranteed
that slit viewer exposures will be taken throughout an entire
science exposure; therefore, it is necessary to be able to compute
the mean exposure epoch (i.e. the effective time that the combined
slit viewer exposures were taken at). This allows a single science
observation to be calibrated using multiple packets of slit viewer
exposures, with appropriate weighting for the time delay between them.
``processSlits`` effectively computes a weighted average of the
exposure epoch of all constituent slit viewer exposures, taking into
account:
- Length of each exposure;
- Whether there is any overlap between the start/end of the
exposure and the start/end of the overall 'image';
- Time of each exposure, relative to the start of the 'image'.
Parameters
----------
suffix: str
suffix to be added to output files
slitflat: str/None
name of the slitflat to use (if None, use the calibration
system)
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
flat_list = params["flat"]
if flat_list is None:
self.getProcessedSlitFlat(adinputs)
flat_list = [
self._get_cal(ad, 'processed_slitflat') for ad in adinputs
]
for ad, slitflat in zip(
*gt.make_lists(adinputs, flat_list, force_ad=True)):
if ad.phu.get(timestamp_key):
log.warning("No changes will be made to {}, since it has "
"already been processed by processSlits".format(
ad.filename))
continue
if slitflat is None:
log.warning("Unable to find slitflat calibration for {}; "
"skipping".format(ad.filename))
continue
else:
sv_flat = slitflat[0].data
# accumulators for computing the mean epoch
sum_of_weights = 0.0
accum_weighted_time = 0.0
# Check the inputs have matching binning and SCI shapes.
try:
gt.check_inputs_match(adinput1=ad,
adinput2=slitflat,
check_filter=False)
except ValueError:
# This is most likely because the science frame has multiple
# extensions and the slitflat needs to be copied
slitflat = gt.clip_auxiliary_data(ad, slitflat, aux_type='cal')
# An Error will be raised if they don't match now
gt.check_inputs_match(ad, slitflat, check_filter=False)
# get science start/end times
sc_start = parse_timestr(ad.phu['UTSTART'])
sc_end = parse_timestr(ad.phu['UTEND'])
res = ad.res_mode()
for ext in ad:
sv_start = parse_timestr(ext.hdr['EXPUTST'])
sv_end = parse_timestr(ext.hdr['EXPUTEND'])
# compute overlap percentage and slit view image duration
latest_start = max(sc_start, sv_start)
earliest_end = min(sc_end, sv_end)
overlap = (earliest_end - latest_start).seconds
overlap = 0.0 if overlap < 0.0 else overlap # no overlap edge case
sv_duration = (sv_end - sv_start).seconds
overlap /= sv_duration # convert into a percentage
# compute the offset (the value to be weighted), in seconds,
# from the start of the science exposure
offset = 42.0 # init value: overridden if overlap, else 0-scaled
if sc_start <= sv_start and sv_end <= sc_end:
offset = (sv_start - sc_start).seconds + sv_duration / 2.0
elif sv_start < sc_start:
offset = overlap * sv_duration / 2.0
elif sv_end > sc_end:
offset = overlap * sv_duration / 2.0
offset += (sv_start - sc_start).seconds
# add flux-weighted offset (plus weight itself) to accumulators
flux = _total_obj_flux(res, ext.data, sv_flat)
weight = flux * overlap
sum_of_weights += weight
accum_weighted_time += weight * offset
# final mean exposure epoch computation
if sum_of_weights > 0.0:
mean_offset = accum_weighted_time / sum_of_weights
mean_offset = timedelta(seconds=mean_offset)
# write the mean exposure epoch into the PHU
sc_start = parse_timestr(ad.phu['UTSTART'])
mean_epoch = sc_start + mean_offset
ad.phu['AVGEPOCH'] = ( # hope this keyword string is ok
mean_epoch.strftime("%H:%M:%S.%f")[:-3],
'Mean Exposure Epoch')
# Timestamp and update filename
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=params["suffix"], strip=True)
return adinputs
def biasCorrect(self, adinputs=None, suffix=None, bias=None, do_cal=None):
"""
The biasCorrect primitive will subtract the science extension of the
input bias frames from the science extension of the input science
frames. The variance and data quality extension will be updated, if
they exist. If no bias is provided, the calibration database(s) will
be queried.
Parameters
----------
suffix: str
suffix to be added to output files
bias: str/list of str
bias(es) to subtract
do_cal: str
perform bias subtraction?
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
if do_cal == 'skip':
log.warning("Bias correction has been turned off.")
return adinputs
if bias is None:
bias_list = self.caldb.get_processed_bias(adinputs)
else:
bias_list = (bias, None)
# Provide a bias AD object for every science frame, and an origin
for ad, bias, origin in zip(*gt.make_lists(adinputs, *bias_list,
force_ad=(1,))):
if ad.phu.get(timestamp_key):
log.warning(f"{ad.filename}: already processed by "
"biasCorrect. Continuing.")
continue
if bias is None:
if 'sq' not in self.mode and do_cal != 'force':
log.warning("No changes will be made to {}, since no "
"bias was specified".format(ad.filename))
continue
else:
log.warning(f"{ad.filename}: no bias was specified. "
"Continuing.")
continue
try:
gt.check_inputs_match(ad, bias, check_filter=False,
check_units=True)
except ValueError:
bias = gt.clip_auxiliary_data(ad, aux=bias, aux_type='cal')
# An Error will be raised if they don't match now
gt.check_inputs_match(ad, bias, check_filter=False,
check_units=True)
origin_str = f" (obtained from {origin})" if origin else ""
log.stdinfo(f"{ad.filename}: subtracting the bias "
f"{bias.filename}{origin_str}")
ad.subtract(bias)
# Record bias used, timestamp, and update filename
ad.phu.set('BIASIM', bias.filename, self.keyword_comments['BIASIM'])
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=suffix, strip=True)
if bias.path:
add_provenance(ad, bias.filename, md5sum(bias.path) or "", self.myself())
timestamp = datetime.now()
return adinputs
def divideByFlat(self, rc):
"""
This primitive will divide each SCI extension of the inputs by those
of the corresponding flat. If the inputs contain VAR or DQ frames,
those will also be updated accordingly due to the division on the data.
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "divideByFlat", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["divideByFlat"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Check for a user-supplied flat
adinput = rc.get_inputs_as_astrodata()
flat_param = rc["flat"]
flat_dict = None
if flat_param is not None:
# The user supplied an input to the flat parameter
if not isinstance(flat_param, list):
flat_list = [flat_param]
else:
flat_list = flat_param
# Convert filenames to AD instances if necessary
tmp_list = []
for flat in flat_list:
if type(flat) is not AstroData:
flat = AstroData(flat)
tmp_list.append(flat)
flat_list = tmp_list
flat_dict = gt.make_dict(key_list=adinput, value_list=flat_list)
# Loop over each input AstroData object in the input list
for ad in adinput:
# Check whether the divideByFlat primitive has been run previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has " \
"already been processed by divideByFlat" \
% (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Retrieve the appropriate flat
if flat_dict is not None:
flat = flat_dict[ad]
else:
flat = rc.get_cal(ad, "processed_flat")
# If there is no appropriate flat, there is no need to divide by
# the flat in QA context; in SQ context, raise an error
if flat is None:
if "qa" in rc.context:
log.warning("No changes will be made to %s, since no " \
"appropriate flat could be retrieved" \
% (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
else:
raise Errors.PrimitiveError("No processed flat found "\
"for %s" % ad.filename)
else:
flat = AstroData(flat)
# Check the inputs have matching filters, binning, and SCI shapes.
try:
gt.check_inputs_match(ad1=ad, ad2=flat)
except Errors.ToolboxError:
# If not, try to clip the flat frame to the size
# of the science data
# For a GMOS example, this allows a full frame flat to
# be used for a CCD2-only science frame.
flat = gt.clip_auxiliary_data(
adinput=ad,aux=flat,aux_type="cal")[0]
# Check again, but allow it to fail if they still don't match
gt.check_inputs_match(ad1=ad, ad2=flat)
# Divide the adinput by the flat
log.fullinfo("Dividing the input AstroData object (%s) " \
"by this flat:\n%s" % (ad.filename,
flat.filename))
ad = ad.div(flat)
# Record the flat file used
ad.phu_set_key_value("FLATIM",
os.path.basename(flat.filename),
comment=self.keyword_comments["FLATIM"])
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def biasCorrect(self, adinputs=None, suffix=None, bias=None, do_bias=True):
"""
The biasCorrect primitive will subtract the science extension of the
input bias frames from the science extension of the input science
frames. The variance and data quality extension will be updated, if
they exist. If no bias is provided, getProcessedBias will be called
to ensure a bias exists for every adinput.
Parameters
----------
suffix: str
suffix to be added to output files
bias: str/list of str
bias(es) to subtract
do_bias: bool
perform bias subtraction?
"""
log = self.log
log.debug(gt.log_message("primitive", self.myself(), "starting"))
timestamp_key = self.timestamp_keys[self.myself()]
if not do_bias:
log.warning("Bias correction has been turned off.")
return adinputs
if bias is None:
self.getProcessedBias(adinputs, refresh=False)
bias_list = self._get_cal(adinputs, 'processed_bias')
else:
bias_list = bias
# Provide a bias AD object for every science frame
for ad, bias in zip(*gt.make_lists(adinputs, bias_list, force_ad=True)):
if ad.phu.get(timestamp_key):
log.warning("No changes will be made to {}, since it has "
"already been processed by biasCorrect".
format(ad.filename))
continue
if bias is None:
if 'qa' in self.mode:
log.warning("No changes will be made to {}, since no "
"bias was specified".format(ad.filename))
continue
else:
raise IOError('No processed bias listed for {}'.
format(ad.filename))
try:
gt.check_inputs_match(ad, bias, check_filter=False,
check_units=True)
except ValueError:
bias = gt.clip_auxiliary_data(ad, aux=bias, aux_type='cal')
# An Error will be raised if they don't match now
gt.check_inputs_match(ad, bias, check_filter=False,
check_units=True)
log.fullinfo('Subtracting this bias from {}:\n{}'.
format(ad.filename, bias.filename))
ad.subtract(bias)
# Record bias used, timestamp, and update filename
ad.phu.set('BIASIM', bias.filename, self.keyword_comments['BIASIM'])
gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key)
ad.update_filename(suffix=suffix, strip=True)
return adinputs
def subtractFringe(self, rc):
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"],
logLevel=rc["logLevel"])
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "subtractFringe",
"starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["subtractFringe"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Check for a user-supplied fringe
adinput = rc.get_inputs_as_astrodata()
fringe_param = rc["fringe"]
fringe_dict = None
if fringe_param is not None:
# The user supplied an input to the fringe parameter
if not isinstance(fringe_param, list):
fringe_list = [fringe_param]
else:
fringe_list = fringe_param
# Convert filenames to AD instances if necessary
tmp_list = []
for fringe in fringe_list:
if type(fringe) is not AstroData:
fringe = AstroData(fringe)
tmp_list.append(fringe)
fringe_list = tmp_list
fringe_dict = gt.make_dict(key_list=adinput, value_list=fringe_list)
# Loop over each input AstroData object in the input list
for ad in adinput:
# Check whether the subtractFringe primitive has been run
# previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has " \
"already been processed by subtractFringe" \
% (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Retrieve the appropriate fringe
if fringe_dict is not None:
fringe = fringe_dict[ad]
else:
fringe = rc.get_cal(ad, "processed_fringe")
# Take care of the case where there was no fringe
if fringe is None:
log.warning("Could not find an appropriate fringe for %s" \
% (ad.filename))
# Append the input to the output without further processing
adoutput_list.append(ad)
continue
else:
fringe = AstroData(fringe)
# Check the inputs have matching filters, binning and SCI shapes.
try:
gt.check_inputs_match(ad1=ad, ad2=fringe)
except Errors.ToolboxError:
# If not, try to clip the fringe frame to the size of the
# science data
# For a GMOS example, this allows a full frame fringe to
# be used for a CCD2-only science frame.
fringe = gt.clip_auxiliary_data(
adinput=ad, aux=fringe, aux_type="cal")[0]
# Check again, but allow it to fail if they still don't match
gt.check_inputs_match(ad1=ad, ad2=fringe)
# Subtract the fringe from the science
ad = ad.sub(fringe)
# Record the fringe file used
ad.phu_set_key_value("FRINGEIM",
os.path.basename(fringe.filename),
comment=self.keyword_comments["FRINGEIM"])
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
def scaleFringeToScience(self, rc):
"""
This primitive will scale the fringes to their matching science data
The fringes should be in the stream this primitive is called on,
and the reference science frames should be loaded into the RC,
as, eg. rc["science"] = adinput.
There are two ways to find the value to scale fringes by:
1. If stats_scale is set to True, the equation:
(letting science data = b (or B), and fringe = a (or A))
arrayB = where({where[SCIb < (SCIb.median+2.5*SCIb.std)]}
> [SCIb.median-3*SCIb.std])
scale = arrayB.std / SCIa.std
The section of the SCI arrays to use for calculating these statistics
is the CCD2 SCI data excluding the outer 5% pixels on all 4 sides.
Future enhancement: allow user to choose section
2. If stats_scale=False, then scale will be calculated using:
exposure time of science / exposure time of fringe
:param stats_scale: Use statistics to calculate the scale values,
rather than exposure time
:type stats_scale: Python boolean (True/False)
"""
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"],
logLevel=rc["logLevel"])
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "scaleFringeToScience",
"starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["scaleFringeToScience"]
# Check for user-supplied science frames
fringe = rc.get_inputs_as_astrodata()
science_param = rc["science"]
fringe_dict = None
if science_param is not None:
# The user supplied an input to the science parameter
if not isinstance(science_param, list):
science_list = [science_param]
else:
science_list = science_param
# If there is one fringe and multiple science frames,
# the fringe must be deepcopied to allow it to be
# scaled separately for each frame
if len(fringe)==1 and len(science_list)>1:
fringe = [deepcopy(fringe[0]) for img in science_list]
# Convert filenames to AD instances if necessary
tmp_list = []
for science in science_list:
if type(science) is not AstroData:
science = AstroData(science)
tmp_list.append(science)
science_list = tmp_list
fringe_dict = gt.make_dict(key_list=science_list,
value_list=fringe)
fringe_output = []
else:
log.warning("No science frames specified; no scaling will be done")
science_list = []
fringe_output = fringe
# Loop over each AstroData object in the science list
for ad in science_list:
# Retrieve the appropriate fringe
fringe = fringe_dict[ad]
# Check the inputs have matching filters, binning and SCI shapes.
try:
gt.check_inputs_match(ad1=ad, ad2=fringe)
except Errors.ToolboxError:
# If not, try to clip the fringe frame to the size of the
# science data
# For a GMOS example, this allows a full frame fringe to
# be used for a CCD2-only science frame.
fringe = gt.clip_auxiliary_data(
adinput=ad, aux=fringe, aux_type="cal")[0]
# Check again, but allow it to fail if they still don't match
gt.check_inputs_match(ad1=ad, ad2=fringe)
# Check whether statistics should be used
stats_scale = rc["stats_scale"]
# Calculate the scale value
scale = 1.0
if not stats_scale:
# Use the exposure times to calculate the scale
log.fullinfo("Using exposure times to calculate the scaling"+
" factor")
try:
scale = ad.exposure_time() / fringe.exposure_time()
except:
raise Errors.InputError("Could not get exposure times " +
"for %s, %s. Try stats_scale=True" %
(ad.filename,fringe.filename))
else:
# Use statistics to calculate the scaling factor
log.fullinfo("Using statistics to calculate the " +
"scaling factor")
# Deepcopy the input so it can be manipulated without
# affecting the original
statsad = deepcopy(ad)
statsfringe = deepcopy(fringe)
# Trim off any overscan region still present
statsad,statsfringe = gt.trim_to_data_section([statsad,
statsfringe])
# Check the number of science extensions; if more than
# one, use CCD2 data only
nsciext = statsad.count_exts("SCI")
if nsciext>1:
# Get the CCD numbers and ordering information
# corresponding to each extension
log.fullinfo("Trimming data to data section to remove "\
"overscan region")
sci_info,frng_info = gt.array_information([statsad,
statsfringe])
# Pull out CCD2 data
scidata = []
frngdata = []
dqdata = []
for i in range(nsciext):
# Get the next extension in physical order
sciext = statsad["SCI",sci_info["amps_order"][i]]
frngext = statsfringe["SCI",frng_info["amps_order"][i]]
# Check to see if it is on CCD2; if so, keep it
if sci_info[
"array_number"][("SCI",sciext.extver())]==2:
scidata.append(sciext.data)
dqext = statsad["DQ",sci_info["amps_order"][i]]
maskext = statsad["OBJMASK",
sci_info["amps_order"][i]]
if dqext is not None and maskext is not None:
dqdata.append(dqext.data | maskext.data)
elif dqext is not None:
dqdata.append(dqext.data)
elif maskext is not None:
dqdata.append(maskext.data)
if frng_info[
"array_number"][("SCI",frngext.extver())]==2:
frngdata.append(frngext.data)
# Stack data if necessary
if len(scidata)>1:
scidata = np.hstack(scidata)
frngdata = np.hstack(frngdata)
else:
scidata = scidata[0]
frngdata = frngdata[0]
if len(dqdata)>0:
if len(dqdata)>1:
dqdata = np.hstack(dqdata)
else:
dqdata = dqdata[0]
else:
dqdata = None
else:
scidata = statsad["SCI"].data
frngdata = statsfringe["SCI"].data
dqext = statsad["DQ"]
maskext = statsad["OBJMASK"]
if dqext is not None and maskext is not None:
dqdata = dqext.data | maskext.data
elif dqext is not None:
dqdata = dqext.data
elif maskext is not None:
dqdata = maskext.data
else:
dqdata = None
if dqdata is not None:
# Replace any DQ-flagged data with the median value
smed = np.median(scidata[dqdata==0])
scidata = np.where(dqdata!=0,smed,scidata)
# Calculate the maximum and minimum in a box centered on
# each data point. The local depth of the fringe is
# max - min. The overall fringe strength is the median
# of the local fringe depths.
# Width of the box is binning and
# filter dependent, determined by experimentation
# Results don't seem to depend heavily on the box size
if ad.filter_name(pretty=True).as_pytype=="i":
size = 20
else:
size = 40
size /= ad.detector_x_bin().as_pytype()
# Use ndimage maximum_filter and minimum_filter to
# get the local maxima and minima
import scipy.ndimage as ndimage
sci_max = ndimage.filters.maximum_filter(scidata,size)
sci_min = ndimage.filters.minimum_filter(scidata,size)
# Take off 5% of the width as a border
xborder = int(0.05 * scidata.shape[1])
yborder = int(0.05 * scidata.shape[0])
if xborder<20:
xborder = 20
if yborder<20:
yborder = 20
sci_max = sci_max[yborder:-yborder,xborder:-xborder]
sci_min = sci_min[yborder:-yborder,xborder:-xborder]
# Take the median difference
sci_df = np.median(sci_max - sci_min)
# Do the same for the fringe
frn_max = ndimage.filters.maximum_filter(frngdata,size)
frn_min = ndimage.filters.minimum_filter(frngdata,size)
frn_max = frn_max[yborder:-yborder,xborder:-xborder]
frn_min = frn_min[yborder:-yborder,xborder:-xborder]
frn_df = np.median(frn_max - frn_min)
# Scale factor
# This tends to overestimate the factor, but it is
# at least in the right ballpark, unlike the estimation
# used in girmfringe (masked_sci.std/fringe.std)
scale = sci_df / frn_df
log.fullinfo("Scale factor found = "+str(scale))
# Use mult from the arith toolbox to perform the scaling of
# the fringe frame
scaled_fringe = fringe.mult(scale)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=scaled_fringe, keyword=timestamp_key)
# Change the filename
scaled_fringe.filename = gt.filename_updater(
adinput=ad, suffix=rc["suffix"], strip=True)
fringe_output.append(scaled_fringe)
# Report the list of output AstroData objects to the reduction context
rc.report_output(fringe_output)
yield rc
