def recover(self):
log.debug("OufileETIFile recover()")
ad = AstroData(self.tmp_name, mode="update")
ad.filename = self.ad_name
ad = gemini_tools.obsmode_del(ad)
log.fullinfo(self.tmp_name + " was loaded into memory")
return ad
def recover(self):
log.debug("OutAtList recover()")
adlist = []
for i, tmpname in enumerate(self.diskoutlist):
ad = AstroData(tmpname, mode="update")
ad.filename = self.ad_name[i]
ad = gemini_tools.obsmode_del(ad)
adlist.append(ad)
log.fullinfo(tmpname + " was loaded into memory")
return adlist
def recover(self):
log.debug("OutAtList recover()")
adlist = []
for i, tmpname in enumerate(self.diskoutlist):
ad = AstroData(tmpname, mode="update")
ad.filename = self.ad_name[i]
ad = gemini_tools.obsmode_del(ad)
# Read the database back in, if it exists
try:
ad = gemini_tools.read_database(
ad, database_name=self.database_name,
input_name=self.tmpin_name[i],
output_name=ad.phu_get_key_value("ORIGNAME"))
except:
pass
adlist.append(ad)
log.fullinfo(tmpname + " was loaded into memory")
return adlist
def test_attr_filename_2():
""" setter """
ad = AstroData(TESTFILE)
ad.filename = "FOO.fits"
assert ad.filename == "FOO.fits"
def _calculate_var(self, adinput=None, add_read_noise=False,
add_poisson_noise=False):
"""
The _calculate_var helper function is used to calculate the variance
and add a variance extension to the single input AstroData object.
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Get the gain and the read noise using the appropriate descriptors.
gain_dv = adinput.gain()
read_noise_dv = adinput.read_noise()
# Only check read_noise here as gain descriptor is only used if units
# are in ADU
if read_noise_dv.is_none() and add_read_noise:
# The descriptor functions return None if a value cannot be found
# and stores the exception info. Re-raise the exception.
if hasattr(adinput, "exception_info"):
raise adinput.exception_info
else:
raise Errors.InputError("read_noise descriptor "
"returned None...\n%s"
% (read_noise_dv.info()))
# Set the data type of the final variance array
var_dtype = np.dtype(np.float32)
# Loop over the science extensions in the dataset
for ext in adinput[SCI]:
extver = ext.extver()
bunit = ext.get_key_value("BUNIT")
if bunit == "adu":
# Get the gain value using the appropriate descriptor. The gain
# is only used if the units are in ADU. Raise if gain is None
gain = gain_dv.get_value(extver=extver)
if gain is not None:
log.fullinfo("Gain for %s[%s,%d] = %f"
% (adinput.filename, SCI, extver, gain))
elif add_read_noise or add_poisson_noise:
err_msg = ("Gain for %s[%s,%d] is None. Cannot calculate "
"variance properly. Setting to zero."
% (adinput.filename, SCI, extver))
raise Errors.InputError(err_msg)
units = "ADU"
elif bunit == "electron" or bunit == "electrons":
units = "electrons"
else:
# Perhaps something more sensible should be done here?
raise Errors.InputError("No units found. Not calculating "
"variance.")
if add_read_noise:
# Get the read noise value (in units of electrons) using the
# appropriate descriptor. The read noise is only used if
# add_read_noise is True
read_noise = read_noise_dv.get_value(extver=extver)
if read_noise is not None:
log.fullinfo("Read noise for %s[%s,%d] = %f"
% (adinput.filename, SCI, extver, read_noise))
# Determine the variance value to use when calculating the
# read noise component of the variance.
read_noise_var_value = read_noise
if units == "ADU":
read_noise_var_value = read_noise / gain
# Add the read noise component of the variance to a zeros
# array that is the same size as the pixel data in the
# science extension
log.fullinfo("Calculating the read noise component of the "
"variance in %s" % units)
var_array_rn = np.add(
np.zeros(ext.data.shape), (read_noise_var_value)**2)
else:
logwarning("Read noise for %s[%s,%d] is None. Setting to "
"zero" % (adinput.filename, SCI, extver))
var_array_rn = np.zeros(ext.data.shape)
if add_poisson_noise:
# Determine the variance value to use when calculating the
# poisson noise component of the variance
poisson_noise_var_value = ext.data
if units == "ADU":
poisson_noise_var_value = ext.data / gain
# Calculate the poisson noise component of the variance. Set
# pixels that are less than or equal to zero to zero.
log.fullinfo("Calculating the poisson noise component of "
"the variance in %s" % units)
var_array_pn = np.where(
ext.data > 0, poisson_noise_var_value, 0)
# Create the final variance array
if add_read_noise and add_poisson_noise:
var_array_final = np.add(var_array_rn, var_array_pn)
if add_read_noise and not add_poisson_noise:
var_array_final = var_array_rn
if not add_read_noise and add_poisson_noise:
var_array_final = var_array_pn
var_array_final = var_array_final.astype(var_dtype)
# If the read noise component and the poisson noise component are
# calculated and added separately, then a variance extension will
# already exist in the input AstroData object. In this case, just
# add this new array to the current variance extension
if adinput[VAR, extver]:
# If both the read noise component and the poisson noise
# component have been calculated, don't add to the variance
# extension
if add_read_noise and add_poisson_noise:
raise Errors.InputError(
"Cannot add read noise component and poisson noise "
"component to variance extension as the variance "
"extension already exists")
else:
log.fullinfo("Combining the newly calculated variance "
"with the current variance extension "
"%s[%s,%d]" % (adinput.filename, VAR, extver))
adinput[VAR, extver].data = np.add(
adinput[VAR, extver].data,
var_array_final).astype(var_dtype)
else:
# Create the variance AstroData object
var = AstroData(data=var_array_final)
var.rename_ext(VAR, ver=extver)
var.filename = adinput.filename
# Call the _update_var_header helper function to update the
# header of the variance extension with some useful keywords
var = self._update_var_header(sci=ext, var=var, bunit=bunit)
# Append the variance AstroData object to the input AstroData
# object.
log.fullinfo("Adding the [%s,%d] extension to the input "
"AstroData object %s" % (VAR, extver,
adinput.filename))
adinput.append(moredata=var)
return adinput
def addDQ(self, rc):
"""
This primitive is used to add a DQ extension to the input AstroData
object. The value of a pixel in the DQ extension will be the sum of the
following: (0=good, 1=bad pixel (found in bad pixel mask), 2=pixel is
in the non-linear regime, 4=pixel is saturated). This primitive will
trim the BPM to match the input AstroData object(s).
:param bpm: The file name, including the full path, of the BPM(s) to be
used to flag bad pixels in the DQ extension. If only one
BPM is provided, that BPM will be used to flag bad pixels
in the DQ extension for all input AstroData object(s). If
more than one BPM is provided, the number of BPMs must
match the number of input AstroData objects. If no BPM is
provided, the primitive will attempt to determine an
appropriate BPM.
:type bpm: string or list of strings
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "addDQ", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["addDQ"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Set the data type of the data quality array
# It can be uint8 for now, it will get converted up as we assign higher bit values
# shouldn't need to force it up to 16bpp yet.
dq_dtype = np.dtype(np.uint8)
#dq_dtype = np.dtype(np.uint16)
# Get the input AstroData objects
adinput = rc.get_inputs_as_astrodata()
# Loop over each input AstroData object in the input list
for ad in adinput:
# Check whether the addDQ primitive has been run previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has "
"already been processed by addDQ" % ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Parameters specified on the command line to reduce are converted
# to strings, including None
##M What about if a user doesn't want to add a BPM at all?
##M Are None's not converted to Nonetype from the command line?
if rc["bpm"] and rc["bpm"] != "None":
# The user supplied an input to the bpm parameter
bpm = rc["bpm"]
else:
# The user did not supply an input to the bpm parameter, so try
# to find an appropriate one. Get the dictionary containing the
# list of BPMs for all instruments and modes.
all_bpm_dict = Lookups.get_lookup_table("Gemini/BPMDict",
"bpm_dict")
# Call the _get_bpm_key helper function to get the key for the
# lookup table
key = self._get_bpm_key(ad)
# Get the appropriate BPM from the look up table
if key in all_bpm_dict:
bpm = lookup_path(all_bpm_dict[key])
else:
bpm = None
log.warning("No BPM found for %s, no BPM will be "
"included" % ad.filename)
# Ensure that the BPMs are AstroData objects
bpm_ad = None
if bpm is not None:
log.fullinfo("Using %s as BPM" % str(bpm))
if isinstance(bpm, AstroData):
bpm_ad = bpm
else:
bpm_ad = AstroData(bpm)
##M Do we want to fail here depending on context?
if bpm_ad is None:
log.warning("Cannot convert %s into an AstroData "
"object, no BPM will be added" % bpm)
final_bpm = None
if bpm_ad is not None:
# Clip the BPM data to match the size of the input AstroData
# object science and pad with overscan region, if necessary
final_bpm = gt.clip_auxiliary_data(adinput=ad, aux=bpm_ad,
aux_type="bpm")[0]
# Get the non-linear level and the saturation level using the
# appropriate descriptors - Individual values get checked in the
# next loop
non_linear_level_dv = ad.non_linear_level()
saturation_level_dv = ad.saturation_level()
# Loop over each science extension in each input AstroData object
for ext in ad[SCI]:
# Retrieve the extension number for this extension
extver = ext.extver()
# Check whether an extension with the same name as the DQ
# AstroData object already exists in the input AstroData object
if ad[DQ, extver]:
log.warning("A [%s,%d] extension already exists in %s"
% (DQ, extver, ad.filename))
continue
# Get the non-linear level and the saturation level for this
# extension
non_linear_level = non_linear_level_dv.get_value(extver=extver)
saturation_level = saturation_level_dv.get_value(extver=extver)
# To store individual arrays created for each of the DQ bit
# types
dq_bit_arrays = []
# Create an array that contains pixels that have a value of 2
# when that pixel is in the non-linear regime in the input
# science extension
if non_linear_level is not None:
non_linear_array = None
if saturation_level is not None:
# Test the saturation level against non_linear level
# They can be the same or the saturation level can be
# greater than but not less than the non-linear level.
# If they are the same then only flag saturated pixels
# below. This just means not creating an unneccessary
# intermediate array.
if saturation_level > non_linear_level:
log.fullinfo("Flagging pixels in the DQ extension "
"corresponding to non linear pixels "
"in %s[%s,%d] using non linear "
"level = %.2f" % (ad.filename, SCI,
extver,
non_linear_level))
non_linear_array = np.where(
((ext.data >= non_linear_level) &
(ext.data < saturation_level)), 2, 0)
elif saturation_level < non_linear_level:
log.warning("%s[%s,%d] saturation_level value is"
"less than the non_linear_level not"
"flagging non linear pixels" %
(ad.filname, SCI, extver))
else:
log.fullinfo("Saturation and non-linear values "
"for %s[%s,%d] are the same. Only "
"flagging saturated pixels."
% (ad.filename, SCI, extver))
else:
log.fullinfo("Flagging pixels in the DQ extension "
"corresponding to non linear pixels "
"in %s[%s,%d] using non linear "
"level = %.2f" % (ad.filename, SCI, extver,
non_linear_level))
non_linear_array = np.where(
(ext.data >= non_linear_level), 2, 0)
dq_bit_arrays.append(non_linear_array)
# Create an array that contains pixels that have a value of 4
# when that pixel is saturated in the input science extension
if saturation_level is not None:
saturation_array = None
log.fullinfo("Flagging pixels in the DQ extension "
"corresponding to saturated pixels in "
"%s[%s,%d] using saturation level = %.2f" %
(ad.filename, SCI, extver, saturation_level))
saturation_array = np.where(
ext.data >= saturation_level, 4, 0)
dq_bit_arrays.append(saturation_array)
# BPMs have an EXTNAME equal to DQ
bpmname = None
if final_bpm is not None:
bpm_array = None
bpmname = os.path.basename(final_bpm.filename)
log.fullinfo("Flagging pixels in the DQ extension "
"corresponding to bad pixels in %s[%s,%d] "
"using the BPM %s[%s,%d]" %
(ad.filename, SCI, extver, bpmname, DQ, extver))
bpm_array = final_bpm[DQ, extver].data
dq_bit_arrays.append(bpm_array)
# Create a single DQ extension from the three arrays (BPM,
# non-linear and saturated)
if not dq_bit_arrays:
# The BPM, non-linear and saturated arrays were not
# created. Create a single DQ array with all pixels set
# equal to 0
log.fullinfo("The BPM, non-linear and saturated arrays "
"were not created. Creating a single DQ "
"array with all the pixels set equal to zero")
final_dq_array = np.zeros(ext.data.shape).astype(dq_dtype)
else:
final_dq_array = self._bitwise_OR_list(dq_bit_arrays)
final_dq_array = final_dq_array.astype(dq_dtype)
# Create a data quality AstroData object
dq = AstroData(data=final_dq_array)
dq.rename_ext(DQ, ver=extver)
dq.filename = ad.filename
# Call the _update_dq_header helper function to update the
# header of the data quality extension with some useful
# keywords
dq = self._update_dq_header(sci=ext, dq=dq, bpmname=bpmname)
# Append the DQ AstroData object to the input AstroData object
log.fullinfo("Adding extension [%s,%d] to %s"
% (DQ, extver, ad.filename))
ad.append(moredata=dq)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
def makeFringeFrame(self,rc):
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"],
logLevel=rc["logLevel"])
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "makeFringeFrame",
"starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Check for at least 3 input frames
adinput = rc.get_inputs_as_astrodata()
if len(adinput)<3:
log.stdinfo('Fewer than 3 frames provided as input. ' +
'Not making fringe frame.')
# Report the empty list to the reduction context
rc.report_output(adoutput_list)
else:
rc.run("correctBackgroundToReferenceImage"\
"(remove_zero_level=True)")
# If needed, do a rough median on all frames, subtract,
# and then redetect to help distinguish sources from fringes
sub_med = rc["subtract_median_image"]
if sub_med:
adinput = rc.get_inputs_as_astrodata()
# Get data by science extension
data = {}
for ad in adinput:
for sciext in ad["SCI"]:
key = (sciext.extname(),sciext.extver())
if data.has_key(key):
data[key].append(sciext.data)
else:
data[key] = [sciext.data]
# Make a median image for each extension
import pyfits as pf
median_ad = AstroData()
median_ad.filename = gt.filename_updater(
adinput=adinput[0], suffix="_stack_median", strip=True)
for key in data:
med_data = np.median(np.dstack(data[key]),axis=2)
hdr = pf.Header()
ext = AstroData(data=med_data, header=hdr)
ext.rename_ext(key)
median_ad.append(ext)
# Subtract the median image
rc["operand"] = median_ad
rc.run("subtract")
# Redetect to get a good object mask
rc.run("detectSources")
# Add the median image back in to the input
rc.run("add")
# Add the object mask into the DQ plane
rc.run("addObjectMaskToDQ")
# Stack frames with masking from DQ plane
rc.run("stackFrames(operation=%s)" % rc["operation"])
yield rc
def test_attr_filename_2():
""" setter """
ad = AstroData(TESTFILE)
ad.filename = "FOO.fits"
assert ad.filename == "FOO.fits"
