def prepare(self):
log.debug("OutDatabase prepare()")
for ad in self.adinput:
inname = gemini_tools.filename_updater(
adinput=ad, prefix=self.get_prefix(), strip=True)
outname = gemini_tools.filename_updater(
adinput=ad, suffix=self.suffix, strip=True)
self.tmpin_name.append(inname)
self.recover_name.append(outname)
self.database_name = "tmpDatabase" + self.pid_task
self.filedict.update({"database": self.database_name})
def prepare(self):
log.debug("InAtList prepare()")
self.database_name = "tmpDatabase" + self.pid_task
log.fullinfo("Temporary database (%s) on disk for the IRAF task %s" %
(self.database_name, self.taskname))
for ad in self.adinput:
ad = gemini_tools.obsmode_add(ad)
newname = gemini_tools.filename_updater(adinput=ad, \
prefix=self.get_prefix(), strip=True)
self.diskinlist.append(newname)
log.fullinfo("Temporary image (%s) on disk for the IRAF task %s" % \
(newname, self.taskname))
ad.write(newname, rename=False, clobber=True)
# Write the wave calibration database record with the
# temporary filename
gemini_tools.write_database(ad, self.database_name, newname)
self.atlist = "tmpImageList" + self.pid_task
fh = open(self.atlist, "w")
for fil in self.diskinlist:
fh.writelines(fil + "\n")
fh.close()
log.fullinfo("Temporary list (%s) on disk for the IRAF task %s" % \
(self.atlist, self.taskname))
self.filedict.update({"inimages": "@" + self.atlist,
"wavtraname": "@" + self.atlist,
"database": self.database_name,})
def storeProcessedFlat(self, rc):
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "storeProcessedFlat",
"starting"))
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Updating the file name with the suffix for this primitive and
# then report the new file to the reduction context
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Adding a PROCFLAT time stamp to the PHU
gt.mark_history(adinput=ad, keyword="PROCFLAT")
# Refresh the AD types to reflect new processed status
ad.refresh_types()
# Upload to cal system
rc.run("storeCalibration")
yield rc
def subtractLampOnLampOff(self, rc):
"""
This primitive subtracts the lamp off stack from the lampon stack. It expects there to be only
one file (the stack) on each stream - call stackLampOnLampOff to do the stacking before calling this
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "subtractLampOnLampOff", "starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
lampon = rc.get_stream(stream="lampOn", style="AD")[0]
lampoff = rc.get_stream(stream="lampOff", style="AD")[0]
log.stdinfo("Lamp ON is: %s %s" % (lampon.data_label(), lampon.filename))
log.stdinfo("Lamp OFF is: %s %s" % (lampoff.data_label(), lampoff.filename))
lampon.sub(lampoff)
lampon.filanme = gt.filename_updater(adinput=lampon, suffix="lampOnOff")
adoutput_list.append(lampon)
rc.report_output(adoutput_list)
yield rc
def prepare(self):
log.debug("Outfile prepare()")
outname = gemini_tools.filename_updater(adinput=self.adinput[0], \
suffix=self.suffix, strip=True)
self.ad_name = outname
self.tmp_name = self.get_prefix() + outname
self.filedict.update({"output": self.tmp_name})
def normalizeFlat(self, rc):
"""
This primitive normalizes each science extension of the input
AstroData object by its mean
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "normalizeFlat", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["normalizeFlat"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Check whether the normalizeFlat primitive has been run previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has " \
"already been processed by normalizeFlat" \
% (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Loop over each science extension in each input AstroData object
for ext in ad[SCI]:
# Normalise the input AstroData object. Calculate the mean
# value of the science extension
mean = np.mean(ext.data, dtype=np.float64)
# Divide the science extension by the mean value of the science
# extension
log.fullinfo("Normalizing %s[%s,%d] by dividing by the mean " \
"= %f" % (ad.filename, ext.extname(),
ext.extver(), mean))
ext = ext.div(mean)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def prepare(self):
log.debug("OutAtList prepare()")
for ad in self.adinput:
outname = gemini_tools.filename_updater(adinput=ad, \
suffix=self.suffix, strip=True)
self.ad_name.append(outname)
# This IRAF task overwrites the file on disk, so use the same
# disk name as for the input list
inname = gemini_tools.filename_updater(adinput=ad, \
prefix=self.get_prefix(), strip=True)
self.diskoutlist.append(inname)
self.atlist = "tmpOutList" + self.pid_task
fh = open(self.atlist, "w")
for fil in self.diskoutlist:
fh.writelines(fil + "\n")
fh.close()
log.fullinfo("Temporary list (%s) on disk for the IRAF task %s" % \
(self.atlist, self.taskname))
def prepare(self):
log.debug("OutAtList prepare()")
self.database_name = "tmpDatabase" + self.pid_task
for ad in self.adinput:
inname = gemini_tools.filename_updater(
adinput=ad, prefix=self.get_prefix(), strip=True)
outname = gemini_tools.filename_updater(adinput=self.adinput[0], \
suffix=self.suffix, strip=True)
self.tmpin_name.append(inname)
self.ad_name.append(outname)
self.diskoutlist.append(self.get_prefix() + outname)
self.atlist = "tmpOutList" + self.pid_task
fh = open(self.atlist, "w")
for fil in self.diskoutlist:
fh.writelines(fil + "\n")
fh.close()
log.fullinfo("Temporary list (%s) on disk for the IRAF task %s" % \
(self.atlist, self.taskname))
self.filedict.update({"outimages": "@" + self.atlist})
def cutFootprints(self, rc):
"""
This primitive will create and append multiple HDU to the output
AD object. Each HDU correspond to a rectangular cut containing a
slit from a MOS Flat exposure or a XD flat exposure as in the
Gnirs case.
:param logLevel: Verbosity setting for log messages to the screen.
:type logLevel: integer from 0-6, 0=nothing to screen, 6=everything to
screen. OR the message level as a string (i.e.,
'critical', 'status', 'fullinfo'...)
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "cutFootprints", "starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Call the user level function
# Check that the input ad has the TRACEFP extension,
# otherwise, create it.
if ad['TRACEFP'] == None:
ad = trace_footprints(ad)
log.stdinfo("Cutting_footprints for: %s"%ad.filename)
try:
adout = cut_footprints(ad)
except:
log.error("Error in cut_slits with file: %s"%ad.filename)
# DO NOT add this input ad to the adoutput_lis
continue
# Change the filename
adout.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects.
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def traceFootprints(self, rc):
"""
This primitive will create and append a 'TRACEFP' Bintable HDU to the
AD object. The content of this HDU is the footprints information
from the espectroscopic flat in the SCI array.
:param logLevel: Verbosity setting for log messages to the screen.
:type logLevel: integer from 0-6, 0=nothing to screen, 6=everything to
screen. OR the message level as a string (i.e.,
'critical', 'status', 'fullinfo'...)
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "", "starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Check whether this primitive has been run previously
if ad.phu_get_key_value("TRACEFP"):
log.warning("%s has already been processed by traceSlits" \
% (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Call the user level function
try:
adout = trace_footprints(ad,
function=rc["function"],
order=rc["order"],
trace_threshold=rc["trace_threshold"])
except:
log.warning("Error in traceFootprints with file: %s" %
ad.filename)
# Change the filename
adout.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects.
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def subtract(self,rc):
# This is a bare-bones primitive interface to the ad sub
# function from the arith module. The value, dictionary,
# or AD instance to be subtracted from the input is stored in
# rc["operand"]
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"],
logLevel=rc["logLevel"])
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "subtract", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["subtract"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Get data to be subtracted from the RC
operand = rc["operand"]
if operand is None:
log.stdinfo("No operand to subtract; no changes will be "\
"made to input")
elif type(operand)==AstroData:
log.stdinfo("Subtracting %s from input" %
(operand.filename))
else:
log.stdinfo("Subtracting %s from input" %
(repr(operand)))
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
if operand is not None:
# Subtract operand from data
ad.sub(operand)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(
adinput=ad, suffix=rc["suffix"], strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def prepare(self):
log.debug("InFile prepare()")
ad = self.adinput[0]
ad = gemini_tools.obsmode_add(ad)
newname = gemini_tools.filename_updater(adinput=ad, \
prefix=self.get_prefix(), strip=True)
self.diskfile = newname
log.fullinfo("Temporary image (%s) on disk for the IRAF task %s" % \
(newname, self.taskname))
ad.write(newname, rename=False, clobber=True)
self.filedict.update({"inimage": self.diskfile})
def addToList(self, rc):
"""
This primitive will update the lists of files to be stacked
that have the same observationID with the current inputs.
This file is cached between calls to reduce, thus allowing
for one-file-at-a-time processing.
:param purpose:
:type purpose: string
"""
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"],
logLevel=rc["logLevel"])
from astrodata import memorytrack as mt
mt.memtrack("addToList [inside]", "1")
# Perform an update to the stack cache file (or create it) using the
# current inputs in the reduction context
purpose = rc["purpose"]
if purpose is None:
purpose = ""
if purpose == "":
suffix = "_list"
else:
suffix = "_%s" % purpose
mt.memtrack("addToList [inside]", "2")
# Update file names and write the files to disk to ensure the right
# version is stored before adding it to the list.
adoutput = []
for ad in rc.get_inputs_as_astrodata():
mt.memtrack("addToList [inside]", "3")
ad.filename = gt.filename_updater(adinput=ad, suffix=suffix,
strip=True)
mt.memtrack("addToList [inside]", "3.1")
log.stdinfo("Writing %s to disk" % ad.filename)
mt.memtrack("addToList [inside]", "3.2")
ad.write(clobber=True)
mt.memtrack("addToList [inside]", "3.3")
adoutput.append(ad)
mt.memtrack("addtoList [inside]", "3.4")
rc.report_output(adoutput)
# Call the rq_stack_update method
rc.rq_stack_update(purpose=purpose)
mt.memtrack("addToList [inside]", "4")
yield rc
def traceFootprints(self, rc):
"""
This primitive will create and append a 'TRACEFP' Bintable HDU to the
AD object. The content of this HDU is the footprints information
from the espectroscopic flat in the SCI array.
:param logLevel: Verbosity setting for log messages to the screen.
:type logLevel: integer from 0-6, 0=nothing to screen, 6=everything to
screen. OR the message level as a string (i.e.,
'critical', 'status', 'fullinfo'...)
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "", "starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Check whether this primitive has been run previously
if ad.phu_get_key_value("TRACEFP"):
log.warning("%s has already been processed by traceSlits" \
% (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Call the user level function
try:
adout = trace_footprints(ad,function=rc["function"],
order=rc["order"],
trace_threshold=rc["trace_threshold"])
except:
log.warning("Error in traceFootprints with file: %s"%ad.filename)
# Change the filename
adout.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects.
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def cutFootprints(self, rc):
"""
This primitive will create and append multiple HDU to the output
AD object. Each HDU correspond to a rectangular cut containing a
slit from a MOS Flat exposure or a XD flat exposure as in the
Gnirs case.
:param logLevel: Verbosity setting for log messages to the screen.
:type logLevel: integer from 0-6, 0=nothing to screen, 6=everything to
screen. OR the message level as a string (i.e.,
'critical', 'status', 'fullinfo'...)
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "cutFootprints", "starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Call the user level function
# Check that the input ad has the TRACEFP extension,
# otherwise, create it.
if ad['TRACEFP'] == None:
ad = trace_footprints(ad)
log.stdinfo("Cutting_footprints for: %s" % ad.filename)
try:
adout = cut_footprints(ad)
except:
log.error("Error in cut_slits with file: %s" % ad.filename)
# DO NOT add this input ad to the adoutput_lis
continue
# Change the filename
adout.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects.
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def wcalResampleToLinearCoords(self,rc):
""" Uses the Wavecal fit_image solution
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Define the keyword to be used for the time stamp
timestamp_key = self.timestamp_keys["wcalResampleToLinearCoords"]
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "wcalResampleToLinearCoords",
"starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Check for a wavelength solution
if ad["WAVECAL"] is None:
if "qa" in rc.context:
log.warning("No wavelength solution found for %s" %
ad.filename)
adout=ad # Don't do anything
else:
raise Errors.InputError("No wavelength solution found "\
"for %s" % ad.filename)
else:
# Wavelength solution found.
wc = Wavecal(ad)
wc.read_wavecal_table()
adout = wc.resample_image_asAstrodata()
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=adout, keyword=timestamp_key)
# Change the filename
adout.filename = gt.filename_updater(
adinput=adout, suffix=rc["suffix"], strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def determineWavelengthSolution(self,rc):
# Instantiate the log
log = logutils.get_logger(__name__)
# Define the keyword to be used for the time stamp
timestamp_key = self.timestamp_keys["determineWavelengthSolution"]
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "determineWavelengthSolution",
"starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Instantiate ETI and then run the task
# Run in a try/except because gswavelength sometimes fails
# badly, and we want to be able to continue without
# wavelength calibration in the QA case
gswavelength_task = eti.gswavelengtheti.GswavelengthETI(rc,ad)
try:
adout = gswavelength_task.run()
except Errors.OutputError:
gswavelength_task.clean()
if "qa" in rc.context:
log.warning("gswavelength failed for input " + ad.filename)
adoutput_list.append(ad)
continue
else:
raise Errors.ScienceError("gswavelength failed for input "+
ad.filename + ". Try interactive"+
"=True")
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=adout, keyword=timestamp_key)
# Change the filename
adout.filename = gt.filename_updater(
adinput=adout, suffix=rc["suffix"], strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def addObjectMaskToDQ(self, rc):
"""
This primitive combines the object mask in a OBJMASK extension
into the DQ plane
"""
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"],
logLevel=rc["logLevel"])
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "addObjectMaskToDQ", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["addObjectMaskToDQ"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
for sciext in ad["SCI"]:
extver = sciext.extver()
dqext = ad["DQ",extver]
mask = ad["OBJMASK",extver]
if mask is None:
log.warning("No object mask present for "\
"%s[SCI,%d]; "\
"cannot apply object mask" %
(ad.filename,extver))
else:
if dqext is not None:
ad["DQ",extver].data = dqext.data | mask.data
else:
dqext = deepcopy(mask)
dqext.rename_ext("DQ",extver)
ad.append(dqext)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def skyCorrectFromSlit(self,rc):
# Instantiate the log
log = logutils.get_logger(__name__)
# Define the keyword to be used for the time stamp
timestamp_key = self.timestamp_keys["skyCorrectFromSlit"]
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "skyCorrectFromSlit", "starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
try:
xbin = ad.detector_x_bin().as_pytype()
ybin = ad.detector_y_bin().as_pytype()
bin_factor = xbin*ybin
roi = ad.detector_roi_setting().as_pytype()
except:
bin_factor = 1
roi = "unknown"
if bin_factor<=2 and roi=="Full Frame" and "qa" in rc.context:
log.warning("Frame is too large to subtract sky efficiently; not "\
"subtracting sky for %s" % ad.filename)
adoutput_list.append(ad)
continue
# Instantiate ETI and then run the task
gsskysub_task = eti.gsskysubeti.GsskysubETI(rc,ad)
adout = gsskysub_task.run()
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=adout, keyword=timestamp_key)
# Change the filename
adout.filename = gt.filename_updater(
adinput=adout, suffix=rc["suffix"], strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def validateData(self, rc):
"""
This primitive is used to validate NIRI data, specifically.
:param repair: Set to True to repair the data, if necessary. Note: this
feature does not work yet.
:type repair: Python boolean
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "validateData", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["validateData"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Check whether the validateData primitive has been run previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has "
"already been processed by validateData"
% ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Validate the input AstroData object.
log.status("No validation required for %s" % ad.filename)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
def prepare(self):
log.debug("InAtList prepare()")
for ad in self.adinput:
ad = gemini_tools.obsmode_add(ad)
newname = gemini_tools.filename_updater(adinput=ad, prefix=self.get_prefix(), strip=True)
self.diskinlist.append(newname)
log.fullinfo("Temporary image (%s) on disk for the IRAF task %s" % (newname, self.taskname))
ad.write(newname, rename=False, clobber=True)
self.atlist = "tmpImageList" + self.pid_task
fh = open(self.atlist, "w")
for fil in self.diskinlist:
fh.writelines(fil + "\n")
fh.close()
log.fullinfo("Temporary list (%s) on disk for the IRAF task %s" % (self.atlist, self.taskname))
self.filedict.update({"inimages": "@" + self.atlist})
def change(self, rc):
inputs = rc.get_inputs_as_astrodata()
# print "pG140:", repr(rc.current_stream), repr(rc._nonstandard_stream)
if rc["changeI"] == None:
rc.update({"changeI":0})
changeI = rc["changeI"]
ci = "_"+str(changeI)
rc.update({"changeI":changeI+1})
for ad in inputs:
ad.filename = gt.filename_updater(adinput=ad, suffix=ci,
strip=False)
# print "pG152:", ad.filename
rc.report_output(inputs)
yield rc
def markAsPrepared(self, rc):
"""
This primitive is used to add a time stamp keyword to the PHU of the
AstroData object and update the AstroData type, allowing the output
AstroData object to be recognised as PREPARED.
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "markAsPrepared", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["prepare"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Update the AstroData type so that the AstroData object is
# recognised as being prepared
ad.refresh_types()
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
def storeProcessedFringe(self, rc):
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "storeProcessedFringe",
"starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Updating the file name with the suffix for this primitive and
# then report the new file to the reduction context
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Sanitize the headers of the file so that it looks like
# a public calibration file rather than a science file
ad = gt.convert_to_cal_header(adinput=ad, caltype="fringe")[0]
# Adding a PROCFRNG time stamp to the PHU
gt.mark_history(adinput=ad, keyword="PROCFRNG")
# Refresh the AD types to reflect new processed status
ad.refresh_types()
adoutput_list.append(ad)
# Upload to cal system
rc.run("storeCalibration")
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def approximateWaveCal(self,rc):
# Instantiate the log
log = logutils.get_logger(__name__)
# Define the keyword to be used for the time stamp
timestamp_key = self.timestamp_keys["appwave"]
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "appwave", "starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
applyApproxWaveCal(ad)
adout = ad
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=adout, keyword=timestamp_key)
# Change the filename
adout.filename = gt.filename_updater(
adinput=adout, suffix=rc["suffix"], strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def attachWavelengthSolution(self,rc):
# Instantiate the log
log = logutils.get_logger(__name__)
# Define the keyword to be used for the time stamp
timestamp_key = self.timestamp_keys["attachWavelengthSolution"]
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "attachWavelengthSolution",
"starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Check for a user-supplied arc
adinput = rc.get_inputs_as_astrodata()
arc_param = rc["arc"]
arc_dict = None
if arc_param is not None:
# The user supplied an input to the arc parameter
if not isinstance(arc_param, list):
arc_list = [arc_param]
else:
arc_list = arc_param
# Convert filenames to AD instances if necessary
tmp_list = []
for arc in arc_list:
if type(arc) is not AstroData:
arc = AstroData(arc)
tmp_list.append(arc)
arc_list = tmp_list
arc_dict = gt.make_dict(key_list=adinput, value_list=arc_list)
for ad in adinput:
if arc_dict is not None:
arc = arc_dict[ad]
else:
arc = rc.get_cal(ad, "processed_arc")
# Take care of the case where there was no arc
if arc is None:
log.warning("Could not find an appropriate arc for %s" \
% (ad.filename))
adoutput_list.append(ad)
continue
else:
arc = AstroData(arc)
wavecal = arc["WAVECAL"]
if wavecal is not None:
# Remove old versions
if ad["WAVECAL"] is not None:
for wc in ad["WAVECAL"]:
ad.remove((wc.extname(),wc.extver()))
# Append new solution
ad.append(wavecal)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
adoutput_list.append(ad)
else:
log.warning("No wavelength solution found for %s" % ad.filename)
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def addVAR(self, rc):
"""
This primitive calculates the variance of each science extension in the
input AstroData object and adds the variance as an additional
extension. This primitive will determine the units of the pixel data in
the input science extension and calculate the variance in the same
units. The two main components of the variance can be calculated and
added separately, if desired, using the following formula:
variance(read_noise) [electrons] = (read_noise [electrons])^2
variance(read_noise) [ADU] = ((read_noise [electrons]) / gain)^2
variance(poisson_noise) [electrons] =
(number of electrons in that pixel)
variance(poisson_noise) [ADU] =
((number of electrons in that pixel) / gain)
The pixel data in the variance extensions will be the same size as the
pixel data in the science extension.
The read noise component of the variance can be calculated and added to
the variance extension at any time, but should be done before
performing operations with other datasets.
The Poisson noise component of the variance can be calculated and added
to the variance extension only after any bias levels have been
subtracted from the pixel data in the science extension.
The variance of a raw bias frame contains only a read noise component
(which represents the uncertainty in the bias level of each pixel),
since the Poisson noise component of a bias frame is meaningless.
:param read_noise: set to True to add the read noise component of the
variance to the variance extension
:type read_noise: Python boolean
:param poisson_noise: set to True to add the Poisson noise component
of the variance to the variance extension
:type poisson_noise: Python boolean
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "addVAR", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["addVAR"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Check to see what component of variance will be added and whether it
# is sensible to do so
read_noise = rc["read_noise"]
poisson_noise = rc["poisson_noise"]
if read_noise and poisson_noise:
log.stdinfo("Adding the read noise component and the poisson "
"noise component of the variance")
if read_noise and not poisson_noise:
log.stdinfo("Adding the read noise component of the variance")
if not read_noise and poisson_noise:
log.stdinfo("Adding the poisson noise component of the variance")
if not read_noise and not poisson_noise:
log.warning("Cannot add a variance extension since no variance "
"component has been selected")
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
if poisson_noise and "BIAS" in ad.types:
log.warning("It is not recommended to add a poisson noise "
"component to the variance of a bias frame")
if (poisson_noise and "GMOS" in ad.types and not
ad.phu_get_key_value(self.timestamp_keys["subtractBias"])):
log.warning("It is not recommended to calculate a poisson "
"noise component of the variance using data that "
"still contains a bias level")
# Call the _calculate_var helper function to calculate and add the
# variance extension to the input AstroData object
ad = self._calculate_var(adinput=ad, add_read_noise=read_noise,
add_poisson_noise=poisson_noise)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
def addDQ(self, rc):
"""
This primitive is used to add a DQ extension to the input AstroData
object. The value of a pixel in the DQ extension will be the sum of the
following: (0=good, 1=bad pixel (found in bad pixel mask), 2=pixel is
in the non-linear regime, 4=pixel is saturated). This primitive will
trim the BPM to match the input AstroData object(s).
:param bpm: The file name, including the full path, of the BPM(s) to be
used to flag bad pixels in the DQ extension. If only one
BPM is provided, that BPM will be used to flag bad pixels
in the DQ extension for all input AstroData object(s). If
more than one BPM is provided, the number of BPMs must
match the number of input AstroData objects. If no BPM is
provided, the primitive will attempt to determine an
appropriate BPM.
:type bpm: string or list of strings
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "addDQ", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["addDQ"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Set the data type of the data quality array
# It can be uint8 for now, it will get converted up as we assign higher bit values
# shouldn't need to force it up to 16bpp yet.
dq_dtype = np.dtype(np.uint8)
#dq_dtype = np.dtype(np.uint16)
# Get the input AstroData objects
adinput = rc.get_inputs_as_astrodata()
# Loop over each input AstroData object in the input list
for ad in adinput:
# Check whether the addDQ primitive has been run previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has "
"already been processed by addDQ" % ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Parameters specified on the command line to reduce are converted
# to strings, including None
##M What about if a user doesn't want to add a BPM at all?
##M Are None's not converted to Nonetype from the command line?
if rc["bpm"] and rc["bpm"] != "None":
# The user supplied an input to the bpm parameter
bpm = rc["bpm"]
else:
# The user did not supply an input to the bpm parameter, so try
# to find an appropriate one. Get the dictionary containing the
# list of BPMs for all instruments and modes.
all_bpm_dict = Lookups.get_lookup_table("Gemini/BPMDict",
"bpm_dict")
# Call the _get_bpm_key helper function to get the key for the
# lookup table
key = self._get_bpm_key(ad)
# Get the appropriate BPM from the look up table
if key in all_bpm_dict:
bpm = lookup_path(all_bpm_dict[key])
else:
bpm = None
log.warning("No BPM found for %s, no BPM will be "
"included" % ad.filename)
# Ensure that the BPMs are AstroData objects
bpm_ad = None
if bpm is not None:
log.fullinfo("Using %s as BPM" % str(bpm))
if isinstance(bpm, AstroData):
bpm_ad = bpm
else:
bpm_ad = AstroData(bpm)
##M Do we want to fail here depending on context?
if bpm_ad is None:
log.warning("Cannot convert %s into an AstroData "
"object, no BPM will be added" % bpm)
final_bpm = None
if bpm_ad is not None:
# Clip the BPM data to match the size of the input AstroData
# object science and pad with overscan region, if necessary
final_bpm = gt.clip_auxiliary_data(adinput=ad, aux=bpm_ad,
aux_type="bpm")[0]
# Get the non-linear level and the saturation level using the
# appropriate descriptors - Individual values get checked in the
# next loop
non_linear_level_dv = ad.non_linear_level()
saturation_level_dv = ad.saturation_level()
# Loop over each science extension in each input AstroData object
for ext in ad[SCI]:
# Retrieve the extension number for this extension
extver = ext.extver()
# Check whether an extension with the same name as the DQ
# AstroData object already exists in the input AstroData object
if ad[DQ, extver]:
log.warning("A [%s,%d] extension already exists in %s"
% (DQ, extver, ad.filename))
continue
# Get the non-linear level and the saturation level for this
# extension
non_linear_level = non_linear_level_dv.get_value(extver=extver)
saturation_level = saturation_level_dv.get_value(extver=extver)
# To store individual arrays created for each of the DQ bit
# types
dq_bit_arrays = []
# Create an array that contains pixels that have a value of 2
# when that pixel is in the non-linear regime in the input
# science extension
if non_linear_level is not None:
non_linear_array = None
if saturation_level is not None:
# Test the saturation level against non_linear level
# They can be the same or the saturation level can be
# greater than but not less than the non-linear level.
# If they are the same then only flag saturated pixels
# below. This just means not creating an unneccessary
# intermediate array.
if saturation_level > non_linear_level:
log.fullinfo("Flagging pixels in the DQ extension "
"corresponding to non linear pixels "
"in %s[%s,%d] using non linear "
"level = %.2f" % (ad.filename, SCI,
extver,
non_linear_level))
non_linear_array = np.where(
((ext.data >= non_linear_level) &
(ext.data < saturation_level)), 2, 0)
elif saturation_level < non_linear_level:
log.warning("%s[%s,%d] saturation_level value is"
"less than the non_linear_level not"
"flagging non linear pixels" %
(ad.filname, SCI, extver))
else:
log.fullinfo("Saturation and non-linear values "
"for %s[%s,%d] are the same. Only "
"flagging saturated pixels."
% (ad.filename, SCI, extver))
else:
log.fullinfo("Flagging pixels in the DQ extension "
"corresponding to non linear pixels "
"in %s[%s,%d] using non linear "
"level = %.2f" % (ad.filename, SCI, extver,
non_linear_level))
non_linear_array = np.where(
(ext.data >= non_linear_level), 2, 0)
dq_bit_arrays.append(non_linear_array)
# Create an array that contains pixels that have a value of 4
# when that pixel is saturated in the input science extension
if saturation_level is not None:
saturation_array = None
log.fullinfo("Flagging pixels in the DQ extension "
"corresponding to saturated pixels in "
"%s[%s,%d] using saturation level = %.2f" %
(ad.filename, SCI, extver, saturation_level))
saturation_array = np.where(
ext.data >= saturation_level, 4, 0)
dq_bit_arrays.append(saturation_array)
# BPMs have an EXTNAME equal to DQ
bpmname = None
if final_bpm is not None:
bpm_array = None
bpmname = os.path.basename(final_bpm.filename)
log.fullinfo("Flagging pixels in the DQ extension "
"corresponding to bad pixels in %s[%s,%d] "
"using the BPM %s[%s,%d]" %
(ad.filename, SCI, extver, bpmname, DQ, extver))
bpm_array = final_bpm[DQ, extver].data
dq_bit_arrays.append(bpm_array)
# Create a single DQ extension from the three arrays (BPM,
# non-linear and saturated)
if not dq_bit_arrays:
# The BPM, non-linear and saturated arrays were not
# created. Create a single DQ array with all pixels set
# equal to 0
log.fullinfo("The BPM, non-linear and saturated arrays "
"were not created. Creating a single DQ "
"array with all the pixels set equal to zero")
final_dq_array = np.zeros(ext.data.shape).astype(dq_dtype)
else:
final_dq_array = self._bitwise_OR_list(dq_bit_arrays)
final_dq_array = final_dq_array.astype(dq_dtype)
# Create a data quality AstroData object
dq = AstroData(data=final_dq_array)
dq.rename_ext(DQ, ver=extver)
dq.filename = ad.filename
# Call the _update_dq_header helper function to update the
# header of the data quality extension with some useful
# keywords
dq = self._update_dq_header(sci=ext, dq=dq, bpmname=bpmname)
# Append the DQ AstroData object to the input AstroData object
log.fullinfo("Adding extension [%s,%d] to %s"
% (DQ, extver, ad.filename))
ad.append(moredata=dq)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
def addMDF(self, rc):
"""
This primitive is used to add an MDF extension to the input AstroData
object. If only one MDF is provided, that MDF will be add to all input
AstroData object(s). If more than one MDF is provided, the number of
MDF AstroData objects must match the number of input AstroData objects.
If no MDF is provided, the primitive will attempt to determine an
appropriate MDF.
:param mdf: The file name of the MDF(s) to be added to the input(s)
:type mdf: string
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "addMDF", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["addMDF"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Get the input AstroData objects
adinput = rc.get_inputs_as_astrodata()
# Loop over each input AstroData object in the input list
for ad in adinput:
# Check whether the addMDF primitive has been run previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has "
"already been processed by addMDF" % ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Check whether the input is spectroscopic data
if "SPECT" not in ad.types:
log.stdinfo("%s is not spectroscopic data, so no MDF will be "
"added" % ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Check whether an MDF extension already exists in the input
# AstroData object
if ad["MDF"]:
log.warning("An MDF extension already exists in %s, so no MDF "
"will be added" % ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Parameters specified on the command line to reduce are converted
# to strings, including None
if rc["mdf"] and rc["mdf"] != "None":
# The user supplied an input to the mdf parameter
mdf = rc["mdf"]
else:
# The user did not supply an input to the mdf parameter, so try
# to find an appropriate one. Get the dictionary containing the
# list of MDFs for all instruments and modes.
all_mdf_dict = Lookups.get_lookup_table("Gemini/MDFDict",
"mdf_dict")
# The MDFs are keyed by the instrument and the MASKNAME. Get
# the instrument and the MASKNAME values using the appropriate
# descriptors
instrument = ad.instrument()
mask_name = ad.phu_get_key_value("MASKNAME")
# Create the key for the lookup table
if instrument is None or mask_name is None:
log.warning("Unable to create the key for the lookup "
"table (%s), so no MDF will be added"
% ad.exception_info)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
key = "%s_%s" % (instrument, mask_name)
# Get the appropriate MDF from the look up table
if key in all_mdf_dict:
mdf = lookup_path(all_mdf_dict[key])
else:
# The MASKNAME keyword defines the actual name of an MDF
if not mask_name.endswith(".fits"):
mdf = "%s.fits" % mask_name
else:
mdf = str(mask_name)
# Check if the MDF exists in the current working directory
if not os.path.exists(mdf):
log.warning("The MDF %s was not found in the current "
"working directory, so no MDF will be "
"added" % mdf)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Ensure that the MDFs are AstroData objects
if not isinstance(mdf, AstroData):
mdf_ad = AstroData(mdf)
if mdf_ad is None:
log.warning("Cannot convert %s into an AstroData object, so "
"no MDF will be added" % mdf)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Check if the MDF is a single extension fits file
if len(mdf_ad) > 1:
log.warning("The MDF %s is not a single extension fits file, "
"so no MDF will be added" % mdf)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Name the extension appropriately
mdf_ad.rename_ext("MDF", 1)
# Append the MDF AstroData object to the input AstroData object
log.fullinfo("Adding the MDF %s to the input AstroData object "
"%s" % (mdf_ad.filename, ad.filename))
ad.append(moredata=mdf_ad)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
def attachWavelengthSolution(self, rc):
# Instantiate the log
log = logutils.get_logger(__name__)
# Define the keyword to be used for the time stamp
timestamp_key = self.timestamp_keys["attachWavelengthSolution"]
# Log the standard "starting primitive" debug message
log.debug(
gt.log_message("primitive", "attachWavelengthSolution",
"starting"))
# Initialize the list of output AstroData objects
adoutput_list = []
# Check for a user-supplied arc
adinput = rc.get_inputs_as_astrodata()
arc_param = rc["arc"]
arc_dict = None
if arc_param is not None:
# The user supplied an input to the arc parameter
if not isinstance(arc_param, list):
arc_list = [arc_param]
else:
arc_list = arc_param
# Convert filenames to AD instances if necessary
tmp_list = []
for arc in arc_list:
if type(arc) is not AstroData:
arc = AstroData(arc)
tmp_list.append(arc)
arc_list = tmp_list
arc_dict = gt.make_dict(key_list=adinput, value_list=arc_list)
for ad in adinput:
if arc_dict is not None:
arc = arc_dict[ad]
else:
arc = rc.get_cal(ad, "processed_arc")
# Take care of the case where there was no arc
if arc is None:
log.warning("Could not find an appropriate arc for %s" \
% (ad.filename))
adoutput_list.append(ad)
continue
else:
arc = AstroData(arc)
wavecal = arc["WAVECAL"]
if wavecal is not None:
# Remove old versions
if ad["WAVECAL"] is not None:
for wc in ad["WAVECAL"]:
ad.remove((wc.extname(), wc.extver()))
# Append new solution
ad.append(wavecal)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad,
primname=self.myself(),
keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
adoutput_list.append(ad)
else:
log.warning("No wavelength solution found for %s" %
ad.filename)
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def mosaicADdetectors(self, rc): # Uses python MosaicAD script
"""
This primitive will mosaic the SCI frames of the input images, along
with the VAR and DQ frames if they exist.
:param tile: tile images instead of mosaic
:type tile: Python boolean (True/False), default is False
"""
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "mosaicADdetectors", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["mosaicADdetectors"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Validate Data
#if (ad.phu_get_key_value('GPREPARE')==None) and \
# (ad.phu_get_key_value('PREPARE')==None):
# raise Errors.InputError("%s must be prepared" % ad.filename)
# Check whether the mosaicDetectors primitive has been run
# previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has " \
"already been processed by mosaicDetectors" \
% (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# If the input AstroData object only has one extension, there is no
# need to mosaic the detectors
if ad.count_exts("SCI") == 1:
log.stdinfo("No changes will be made to %s, since it " \
"contains only one extension" % (ad.filename))
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Get the necessary parameters from the RC
tile = rc["tile"]
log.stdinfo("Mosaicking %s ..." % ad.filename)
log.stdinfo("MosaicAD: Using tile: %s ..." % tile)
#t1 = time.time()
mo = MosaicAD(ad,
mosaic_ad_function=gemini_mosaic_function,
dq_planes=rc['dq_planes'])
adout = mo.as_astrodata(tile=tile)
#t2 = time.time()
#print '%s took %0.3f ms' % ('as_astrodata', (t2-t1)*1000.0)
# Verify mosaicAD was actually run on the file
# then log file names of successfully reduced files
if adout.phu_get_key_value("MOSAIC"):
log.fullinfo("File "+adout.filename+\
" was successfully mosaicked")
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=adout,
primname=self.myself(),
keyword=timestamp_key)
# Change the filename
adout.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list
# of output AstroData objects
adoutput_list.append(adout)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def correctWCSToReferenceFrame(self, rc):
"""
This primitive registers images to a reference image by correcting
the relative error in their world coordinate systems. The function
uses points of reference common to the reference image and the
input images to fit the input WCS to the reference one. The fit
is done by a least-squares minimization of the difference between
the reference points in the input image pixel coordinate system.
This function is intended to be followed by the
align_to_reference_image function, which applies the relative
transformation encoded in the WCS to transform input images into the
reference image pixel coordinate system.
The primary registration method is intended to be by direct mapping
of sources in the image frame to correlated sources in the reference
frame. This method fails when there are no correlated sources in the
field, or when the WCSs are very far off to begin with. As a back-up
method, the user can try correcting the WCS by the shifts indicated
in the POFFSET and QOFFSET header keywords (option fallback='header'),
By default, only the direct method is
attempted, as it is expected that the relative WCS will generally be
more correct than either indirect method. If the user prefers not to
attempt direct mapping at all, they may set method to 'header'.
In order to use the direct mapping method, sources must have been
detected in the frame and attached to the AstroData instance in an
OBJCAT extension. This can be accomplished via the detectSources
primitive. Running time is optimal, and sometimes the solution is
more robust, when there are not too many sources in the OBJCAT. Try
running detectSources with threshold=20. The solution may also be
more robust if sub-optimal sources are rejected from the set of
correlated sources (use option cull_sources=True). This option may
substantially increase the running time if there are many sources in
the OBJCAT.
It is expected that the relative difference between the WCSs of
images to be combined should be quite small, so it may not be necessary
to allow rotation and scaling degrees of freedom when fitting the image
WCS to the reference WCS. However, if it is desired, the options
rotate and scale can be used to allow these degrees of freedom. Note
that these options refer to rotation/scaling of the WCS itself, not the
images. Significant rotation and scaling of the images themselves
will generally already be encoded in the WCS, and will be corrected for
when the images are aligned.
The WCS keywords in the headers of the output images are updated
to contain the optimal registration solution.
:param method: method to use to generate reference points. Options
are 'sources' to directly map sources from the input
image to the reference image,
or 'header' to generate reference points from the
POFFSET and QOFFSET keywords in the image headers.
:type method: string, either 'sources' or 'header'
:param fallback: back-up method for generating reference points.
if the primary method fails. The 'sources' option
cannot be used as the fallback.
:type fallback: string, either 'header' or None.
:param cull_sources: flag to indicate whether sub-optimal sources
should be rejected before attempting a direct
mapping. If True, sources that are saturated,
or otherwise unlikely to be point sources
will be eliminated from the list of reference
points.
:type cull_sources: bool
:param rotate: flag to indicate whether the input image WCSs should
be allowed to rotate with respect to the reference image
WCS
:type rotate: bool
:param scale: flag to indicate whether the input image WCSs should
be allowed to scale with respect to the reference image
WCS. The same scale factor is applied to all dimensions.
:type scale: bool
"""
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"], logLevel=rc["logLevel"])
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "correctWCSToReferenceFrame", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["correctWCSToReferenceFrame"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Check whether two or more input AstroData objects were provided
adinput = rc.get_inputs_as_astrodata()
correcting = True
if len(adinput) <= 1:
log.warning(
"No correction will be performed, since at least "
"two input AstroData objects are required for "
"correctWCSToReferenceFrame"
)
# Set the input AstroData object list equal to the output AstroData
# objects list without further processing
adoutput_list = adinput
correcting = False
# Check that method is sensibly defined
if correcting:
# Get the necessary parameters from the RC
method = rc["method"]
fallback = rc["fallback"]
cull_sources = rc["cull_sources"]
rotate = rc["rotate"]
scale = rc["scale"]
if method == "None":
method = None
if fallback == "None":
fallback = None
if method is None:
if fallback is None:
log.warning("No correction will be performed, since both " "method and fallback are None")
adoutput_list = adinput
correcting = False
else:
method = fallback
# Check that images have one SCI extension, and if necessary,
# sources defined in an OBJCAT extension
if correcting:
n_test = []
for ad in adinput:
# Make sure all images have one science extension
if len(ad["SCI"]) != 1:
raise Errors.InputError("Input images must have only one " "SCI extension.")
# Get number of objects from OBJCAT
objcat = ad["OBJCAT"]
if objcat is None:
num_cat = 0
else:
num_cat = len(objcat)
if num_cat == 0:
n_obj = 0
elif num_cat > 1:
raise Errors.InputError("Input images must have only one " + "OBJCAT extension.")
else:
n_obj = len(objcat.data)
n_test.append(n_obj)
if n_test[0] == 0 and method == "sources":
log.warning("No objects found in reference image.")
if fallback is not None:
log.warning("Only attempting indirect WCS alignment, " + "via " + fallback + " mapping")
method = fallback
else:
log.warning(
"WCS can only be corrected indirectly "
+ "and fallback method is set to None. Not "
+ "attempting WCS correction."
)
adoutput_list = adinput
correcting = False
# If input passed all checks, apply the selected method
if correcting:
# Reference image is first one supplied
# (won't be modified)
reference = adinput[0]
adoutput_list.append(reference)
log.stdinfo("Reference image: " + reference.filename)
# If no OBJCAT/no sources in reference image, or user choice,
# use indirect alignment for all images at once
if method == "header":
reg_ad = _header_align(reference, adinput[1:])
adoutput_list.extend(reg_ad)
elif method != "sources":
raise Errors.InputError("Did not recognize method " + method)
# otherwise try to do direct alignment for each image by correlating
# sources in the reference and input images
else:
for i in range(1, len(adinput)):
ad = adinput[i]
if n_test[i] == 0:
log.warning("No objects found in " + ad.filename)
if fallback is not None:
log.warning("Only attempting indirect WCS alignment, " + "via " + fallback + " mapping")
if fallback == "header":
adoutput = _header_align(reference, ad)
else:
raise Errors.InputError("Did not recognize fallback method " + fallback)
else:
log.warning(
"WCS can only be corrected indirectly "
+ "and fallback=None. Not attempting WCS "
+ "correction for "
+ ad.filename
)
adoutput_list.append(ad)
continue
else:
log.fullinfo("Number of objects in image %s: %d" % (ad.filename, n_test[i]))
log.fullinfo("Cross-correlating sources in %s, %s" % (reference.filename, ad.filename))
obj_list = _correlate_sources(reference, ad, cull_sources=cull_sources)
n_corr = len(obj_list[0])
if n_corr == 0:
log.warning("No correlated sources found.")
if fallback is not None:
log.warning("Only attempting indirect WCS " + "alignment, via " + fallback + " mapping")
if fallback == "header":
adoutput = _header_align(reference, ad)
else:
raise Errors.InputError("Did not recognize " + "fallback method " + fallback)
else:
log.warning(
"WCS can only be corrected indirectly "
+ "and fallback=None. Not attempting "
+ "WCS correction for "
+ ad.filename
)
adoutput_list.append(ad)
continue
else:
log.fullinfo("Number of correlated sources: %d" % n_corr)
# Check the fit geometry depending on the
# number of objects
if n_corr == 1:
log.warning("Too few objects. Setting " + "rotate=False, " + "scale=False")
rotate = False
scale = False
log.fullinfo("\nSources used to align frames:")
log.fullinfo(
" %7s %7s %7s %7s\n%s" % (" Ref. x", "Ref. y", "Img. x", "Img. y", " " + "-" * 31)
)
output_obj = zip(obj_list[0], obj_list[1])
for obj in output_obj:
obj_string = " %7.2f %7.2f %7.2f %7.2f" % (obj[0][0], obj[0][1], obj[1][0], obj[1][1])
log.fullinfo(obj_string)
log.fullinfo("")
adoutput = _align_wcs(reference, ad, [obj_list], rotate=rotate, scale=scale)
adoutput_list.extend(adoutput)
# Change the filenames and add the appropriate timestamps
for ad in adoutput_list:
gt.mark_history(adinput=ad, keyword=timestamp_key)
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"], strip=True)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def standardizeGeminiHeaders(self, rc):
"""
This primitive is used to make the changes and additions to the
keywords in the headers of Gemini data.
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(
gt.log_message("primitive", "standardizeGeminiHeaders",
"starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["standardizeGeminiHeaders"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Check whether the standardizeGeminiHeaders primitive has been run
# previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has "
"already been processed by "
"standardizeGeminiHeaders" % ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Standardize the headers of the input AstroData object. Update the
# keywords in the headers that are common to all Gemini data.
log.status("Updating keywords that are common to all Gemini data")
# Original name
ad.store_original_name()
# Number of science extensions
gt.update_key(adinput=ad,
keyword="NSCIEXT",
value=ad.count_exts("SCI"),
comment=None,
extname="PHU",
keyword_comments=self.keyword_comments)
# Number of extensions
gt.update_key(adinput=ad,
keyword="NEXTEND",
value=len(ad),
comment=None,
extname="PHU",
keyword_comments=self.keyword_comments)
# Physical units (assuming raw data has units of ADU)
gt.update_key(adinput=ad,
keyword="BUNIT",
value="adu",
comment=None,
extname="SCI",
keyword_comments=self.keyword_comments)
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad,
primname=self.myself(),
keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad,
suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
def updateWCS(self, rc):
"""
This primitive applies a previously calculated WCS correction.
The solution should be stored in the RC as a dictionary, with
astrodata instances as the keys and pywcs.WCS objects as the
values.
"""
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"], logLevel=rc["logLevel"])
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "updateWCS", "starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["updateWCS"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Get the necessary parameters from the RC
wcs = rc["wcs"]
if wcs is None:
log.warning("No new WCS supplied; no correction will be " "performed.")
else:
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
ad_wcs = None
if isinstance(wcs, dict):
try:
ad_wcs = wcs[ad]
except KeyError:
ad_wcs = wcs
elif isinstance(wcs, pywcs.WCS):
ad_wcs = wcs
if ad_wcs is None:
log.warning("No new WCS supplied for %s; " "no correction will be performed" % ad.filename)
adoutput_list.append(ad)
continue
for ext in ad:
extname = ext.extname()
extver = ext.extver()
ext_wcs = None
if isinstance(ad_wcs, dict):
try:
ext_wcs = ad_wcs[extver]
except KeyError:
pass
elif isinstance(ad_wcs, pywcs.WCS):
ext_wcs = wcs
if ext_wcs is None:
log.warning(
"No new WCS supplied for %s[%s,%d]; "
"no correction will be performed" % (ad.filename, extname, extver)
)
continue
elif not isinstance(ext_wcs, pywcs.WCS):
raise Errors.InputError(
"Parameter wcs must be "
"either a pywcs.WCS object "
"or a dictionary of pywcs.WCS "
"objects"
)
# If image extension, correct the header values
if extname in ["SCI", "VAR", "DQ"]:
log.fullinfo(
"Correcting CRVAL, CRPIX, and CD in "
"image extension headers for %s[%s,%d]" % (ad.filename, extname, extver)
)
log.fullinfo("CRVAL: " + repr(ext_wcs.wcs.crval))
log.fullinfo("CRPIX: " + repr(ext_wcs.wcs.crpix))
log.fullinfo("CD: " + repr(ext_wcs.wcs.cd))
ext.set_key_value("CRVAL1", ext_wcs.wcs.crval[0], comment=keyword_comments["CRVAL1"])
ext.set_key_value("CRVAL2", ext_wcs.wcs.crval[1], comment=keyword_comments["CRVAL2"])
ext.set_key_value("CRPIX1", ext_wcs.wcs.crpix[0], comment=keyword_comments["CRPIX1"])
ext.set_key_value("CRPIX2", ext_wcs.wcs.crpix[1], comment=keyword_comments["CRPIX2"])
ext.set_key_value("CD1_1", ext_wcs.wcs.cd[0, 0], comment=keyword_comments["CD1_1"])
ext.set_key_value("CD1_2", ext_wcs.wcs.cd[0, 1], comment=keyword_comments["CD1_2"])
ext.set_key_value("CD2_1", ext_wcs.wcs.cd[1, 0], comment=keyword_comments["CD2_1"])
ext.set_key_value("CD2_2", ext_wcs.wcs.cd[1, 1], comment=keyword_comments["CD2_2"])
# If objcat, fix the RA/Dec columns
elif extname == "OBJCAT":
log.fullinfo(
"Correcting RA, Dec columns in OBJCAT "
"extension for %s[%s,%d]" % (ad.filename, extname, extver)
)
for row in ext.data:
xy = np.array([row["X_IMAGE"], row["Y_IMAGE"]])
radec = ext_wcs.wcs_pix2sky([xy], 1)[0]
# FIXME - is it correct to set oring to 1 here?
# Also we should be setting ra_dec_order=True, but
# that breaks with the wcs missing the lattype
# property
row["X_WORLD"] = radec[0]
row["Y_WORLD"] = radec[1]
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"], strip=True)
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction
# context
rc.report_output(adoutput_list)
yield rc
def standardizeGeminiHeaders(self, rc):
"""
This primitive is used to make the changes and additions to the
keywords in the headers of Gemini data.
"""
# Instantiate the log
log = logutils.get_logger(__name__)
# Log the standard "starting primitive" debug message
log.debug(gt.log_message("primitive", "standardizeGeminiHeaders",
"starting"))
# Define the keyword to be used for the time stamp for this primitive
timestamp_key = self.timestamp_keys["standardizeGeminiHeaders"]
# Initialize the list of output AstroData objects
adoutput_list = []
# Loop over each input AstroData object in the input list
for ad in rc.get_inputs_as_astrodata():
# Check whether the standardizeGeminiHeaders primitive has been run
# previously
if ad.phu_get_key_value(timestamp_key):
log.warning("No changes will be made to %s, since it has "
"already been processed by "
"standardizeGeminiHeaders" % ad.filename)
# Append the input AstroData object to the list of output
# AstroData objects without further processing
adoutput_list.append(ad)
continue
# Standardize the headers of the input AstroData object. Update the
# keywords in the headers that are common to all Gemini data.
log.status("Updating keywords that are common to all Gemini data")
# Original name
ad.store_original_name()
# Number of science extensions
gt.update_key(adinput=ad, keyword="NSCIEXT",
value=ad.count_exts("SCI"), comment=None,
extname="PHU")
# Number of extensions
gt.update_key(adinput=ad, keyword="NEXTEND", value=len(ad),
comment=None, extname="PHU")
# Physical units (assuming raw data has units of ADU)
gt.update_key(adinput=ad, keyword="BUNIT", value="adu",
comment=None, extname="SCI")
# Add the appropriate time stamps to the PHU
gt.mark_history(adinput=ad, keyword=timestamp_key)
# Change the filename
ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
strip=True)
# Append the output AstroData object to the list of output
# AstroData objects
adoutput_list.append(ad)
# Report the list of output AstroData objects to the reduction context
rc.report_output(adoutput_list)
yield rc
