def __init__(self, prop_id, obset_id, instrument, detector, filename, filters, filetype, log_level):
super().__init__(prop_id, obset_id, instrument, detector, filename, filetype, log_level)
self.info = '_'.join([prop_id, obset_id, instrument, detector, filename, filters, filetype])
self.filters = filters
self.full_filename = self.copy_exposure(filename)
# Open the input FITS file to mine some header information.
# and make sure the WCS is up-to-date
hdu_list = fits.open(filename)
self.mjdutc = hdu_list[0].header['EXPSTART']
self.exptime = hdu_list[0].header['EXPTIME']
drizcorr = hdu_list[0].header['DRIZCORR']
if drizcorr == "OMIT":
updatewcs.updatewcs(self.full_filename, use_db=True)
hdu_list.close()
self.product_basename = self.basename + "_".join(map(str, [filters, self.exposure_name]))
self.drizzle_filename = self.product_basename + "_" + self.filetype + ".fits"
self.headerlet_filename = self.product_basename + "_hlet.fits"
self.trl_logname = self.product_basename + "_trl.log"
self.trl_filename = self.product_basename + "_trl.txt"
self.regions_dict = {}
# Define HAPLEVEL value for this product
self.haplevel = 1
log.info("Grism Exposure object {} created.".format(self.full_filename))
def initImage(fitsfile):
"""Prepare an image for pipeline run
- Checks if ACSWFC NPOL needs updating
- Checks if moving target
- Returns metadata (instdet, filter)
"""
instdet = utils.getInstDet(fitsfile)
hdr = fits.getheader(fitsfile)
if instdet == 'acswfc':
if not 'SIPNAME' in hdr.keys():
print 'Updating NPOL'
updatenpol.update(fitsfile,'jref$')
updatewcs.updatewcs(fitsfile)
if instdet == 'wfc3ir' or instdet == 'wfc3uvis':
if 'WCSAXESO' not in hdr.keys():
print 'Updating WCS'
updatewcs.updatewcs(fitsfile)
print 'Checking if moving target observation'
with fits.open(fitsfile, mode='update') as hdu:
hdr = hdu[0].header
if hdr['mtflag'] == 'T':
hdr['mtflag'] = ' '
filter = utils.getFilter(fitsfile)
return instdet, filter
def test_align_47tuc(self):
""" Verify whether NGC188 exposures can be aligned to an astrometric standard.
Characeteristics of this test:
* Input exposures include both ACS and WFC3 images of the same general field-of-view
of 47Tuc suitable for creating a combined mosaic using both instruments.
"""
self.input_filenames = [
'ib6v06c4q_flc.fits', 'ib6v06c7q_flc.fits', 'ib6v25aqq_flc.fits',
'ib6v25atq_flc.fits', 'jddh02gjq_flc.fits', 'jddh02glq_flc.fits',
'jddh02goq_flc.fits'
]
self.input_loc = 'mosaic_47tuc'
for infile in self.input_filenames:
self.get_input_file(infile, docopy=True)
updatewcs.updatewcs(infile)
output_shift_file = 'test_mosaic_47tuc_shifts.txt'
align_to_gaia.align(self.input_filenames, shift_name=output_shift_file)
shift_file = Table.read(output_shift_file, format='ascii')
rms_x = max(shift_file['col6'])
rms_y = max(shift_file['col7'])
assert (rms_x <= 0.25 and rms_y <= 0.25)
def copy_adriz_headerlets(direct_asn='GOODS-S-15-F140W_asn.fits', grism_asn='GOODS-S-15-G141_asn.fits', order=None, force=False, ACS=False):
"""
Copy Tweaked WCS solution in direct image to the paired grism exposures.
If same number of grism as direct exposures, match the WCS headers
directly. If not, just get the overall shift from the first direct
exposure and apply that to the grism exposures.
"""
import stwcs
from stwcs import updatewcs
import drizzlepac
direct = threedhst.utils.ASNFile(direct_asn)
grism = threedhst.utils.ASNFile(grism_asn)
Nd = len(direct.exposures)
Ng = len(grism.exposures)
if ACS:
NCHIP=2
sci_ext = [1,4]
ext = 'flc'
else:
NCHIP=1
sci_ext = [1]
ext = 'flt'
if Nd == Ng:
if order is None:
order = range(Nd)
for i in range(Nd):
imd = pyfits.open('%s_%s.fits' %(direct.exposures[i], ext))
#img = pyfits.open('%s_%s.fits' %(grism.exposures[i]))
#
for sci in sci_ext:
#sci_ext=1
direct_WCS = stwcs.wcsutil.HSTWCS(imd, ext=sci)
#
drizzlepac.updatehdr.update_wcs('%s_%s.fits' %(grism.exposures[order[i]], ext), sci, direct_WCS, verbose=True)
else:
#### Get overall shift from a shift-file and apply it to the
#### grism exposures
sf = threedhst.shifts.ShiftFile(direct_asn.replace('_asn.fits', '_shifts.txt'))
imd = pyfits.open(direct_asn.replace('asn','wcs'))
print imd.filename()
direct_WCS = stwcs.wcsutil.HSTWCS(imd, ext='wcs')
#
for i in range(Ng):
img = pyfits.open('%s_%s.fits' %(grism.exposures[i], ext))
if 'WCSNAME' in img[1].header:
if img[1].header['WCSNAME'] == 'TWEAK':
if force is False:
threedhst.showMessage('"TWEAK" WCS already found in %s_flt.fits.\nRun copy_adriz_headerlets with force=True to force update the shifts' %(grism.exposures[i]), warn=True)
continue
#
updatewcs.updatewcs('%s_%s.fits' %(grism.exposures[i], ext))
drizzlepac.updatehdr.updatewcs_with_shift('%s_%s.fits' %(grism.exposures[i], ext), direct_WCS, rot=sf.rotate[0], scale=sf.scale[0], xsh=sf.xshift[0], ysh=sf.yshift[0], wcsname='TWEAK')
def buildFileListOrig(input, output=None, ivmlist=None,
wcskey=None, updatewcs=True, **workinplace):
"""
Builds a file list which has undergone various instrument-specific
checks for input to MultiDrizzle, including splitting STIS associations.
Compared to buildFileList, this version returns the list of the
original file names as specified by the user (e.g., before GEIS->MEF, or
WAIVER FITS->MEF conversion).
"""
# NOTE: original file name is required in order to correctly associate
# user catalog files (e.g., user masks to be used with 'skymatch') with
# corresponding imageObjects.
filelist, output, ivmlist, oldasndict = processFilenames(input,output)
# verify that all input images specified can be updated as needed
filelist = util.verifyFilePermissions(filelist)
if filelist is None or len(filelist) == 0:
return None, None, None, None, None
manageInputCopies(filelist,**workinplace)
# to keep track of the original file names we do the following trick:
# pack filelist with the ivmlist using zip and later unpack the zipped list.
#
# NOTE: this required a small modification of the checkStisFiles function
# in stsci.tools.check_files to be able to handle ivmlists that are tuples.
if ivmlist is None:
ivmlist = len(filelist)*[None]
else:
assert(len(filelist) == len(ivmlist)) #TODO: remove after debugging
ivmlist = list(zip(ivmlist,filelist))
# Check format of FITS files - convert Waiver/GEIS to MEF if necessary
filelist, ivmlist = check_files.checkFITSFormat(filelist, ivmlist)
# check for non-polynomial distortion correction
if not updatewcs:
# with updatewcs turned on, any problems will get resolved
# so we do not need to be concerned about the state of the DGEOFILEs
filelist = checkDGEOFile(filelist)
# run all WCS updating
updated_input = _process_input_wcs(filelist, wcskey, updatewcs)
newfilelist, ivmlist = check_files.checkFiles(updated_input, ivmlist)
if updatewcs:
uw.updatewcs(','.join(set(newfilelist) - set(filelist)))
if len(ivmlist) > 0:
ivmlist, filelist = list(zip(*ivmlist))
else:
filelist = [] # insure that both filelist and ivmlist are defined as empty lists
return newfilelist, ivmlist, output, oldasndict, filelist
def buildFileListOrig(input, output=None, ivmlist=None,
wcskey=None, updatewcs=True, **workinplace):
"""
Builds a file list which has undergone various instrument-specific
checks for input to MultiDrizzle, including splitting STIS associations.
Compared to buildFileList, this version returns the list of the
original file names as specified by the user (e.g., before GEIS->MEF, or
WAIVER FITS->MEF conversion).
"""
# NOTE: original file name is required in order to correctly associate
# user catalog files (e.g., user masks to be used with 'skymatch') with
# corresponding imageObjects.
filelist, output, ivmlist, oldasndict = processFilenames(input,output)
# verify that all input images specified can be updated as needed
filelist = util.verifyFilePermissions(filelist)
if filelist is None or len(filelist) == 0:
return None, None, None, None, None
manageInputCopies(filelist,**workinplace)
# to keep track of the original file names we do the following trick:
# pack filelist with the ivmlist using zip and later unpack the zipped list.
#
# NOTE: this required a small modification of the checkStisFiles function
# in stsci.tools.check_files to be able to handle ivmlists that are tuples.
if ivmlist is None:
ivmlist = len(filelist)*[None]
else:
assert(len(filelist) == len(ivmlist)) #TODO: remove after debugging
ivmlist = list(zip(ivmlist,filelist))
# Check format of FITS files - convert Waiver/GEIS to MEF if necessary
filelist, ivmlist = check_files.checkFITSFormat(filelist, ivmlist)
# check for non-polynomial distortion correction
if not updatewcs:
# with updatewcs turned on, any problems will get resolved
# so we do not need to be concerned about the state of the DGEOFILEs
filelist = checkDGEOFile(filelist)
# run all WCS updating
updated_input = _process_input_wcs(filelist, wcskey, updatewcs)
newfilelist, ivmlist = check_files.checkFiles(updated_input, ivmlist)
if updatewcs:
uw.updatewcs(','.join(set(newfilelist) - set(filelist)))
if len(ivmlist) > 0:
ivmlist, filelist = list(zip(*ivmlist))
else:
filelist = [] # insure that both filelist and ivmlist are defined as empty lists
return newfilelist, ivmlist, output, oldasndict, filelist
def run(configObj=None):
# Interpret primary parameters from configObj instance
extname = configObj['extname']
input = configObj['input']
# create dictionary of remaining parameters, deleting extraneous ones
# such as those above
cdict = configObj.dict()
# remove any rules defined for the TEAL interface
if "_RULES_" in cdict: del cdict['_RULES_']
del cdict['_task_name_']
del cdict['input']
del cdict['extname']
# parse input
input, altfiles = parseinput.parseinput(configObj['input'])
# Insure that all input files have a correctly archived
# set of OPUS WCS keywords
# Legacy files from OTFR, like all WFPC2 data from OTFR, will only
# have the OPUS WCS keywords archived using a prefix of 'O'
# These keywords need to be converted to the Paper I alternate WCS
# standard using a wcskey (suffix) of 'O'
# If an alternate WCS with wcskey='O' already exists, this will copy
# the values from the old prefix-'O' WCS keywords to insure the correct
# OPUS keyword values get archived for use with updatewcs.
#
for file in input:
# Check to insure that there is a valid reference file to be used
idctab = fits.getval(file, 'idctab')
if not os.path.exists(fileutil.osfn(idctab)):
print('No valid distortion reference file ', idctab, ' found in ',
file, '!')
raise ValueError
# Re-define 'cdict' to only have switches for steps supported by that instrument
# the set of supported steps are defined by the dictionary
# updatewcs.apply_corrections.allowed_corrections
#
for file in input:
# get instrument name from input file
instr = fits.getval(file, 'INSTRUME')
# make copy of input parameters dict for this file
fdict = cdict.copy()
# Remove any parameter that is not part of this instrument's allowed corrections
for step in allowed_corr_dict:
if allowed_corr_dict[
step] not in updatewcs.apply_corrections.allowed_corrections[
instr]:
fdict[step]
# Call 'updatewcs' on correctly archived file
updatewcs.updatewcs(file, **fdict)
def run_mtpipeline(root_filename, output_path = None, cr_reject_switch=True,
astrodrizzle_switch=True, png_switch=True, reproc_switch=False):
'''
This is the main controller for all the steps in the pipeline.
'''
# Generate the output drizzle names
filename = os.path.abspath(root_filename)
output_file_dict = make_output_file_dict(root_filename)
# Run CR reject.
if cr_reject_switch:
output_check = check_for_outputs(output_file_dict['cr_reject_output'][1])
if reproc_switch == False and output_check == True:
print 'Not reprocessing cr_reject files.'
else:
print 'Running cr_reject'
run_cosmics(root_filename, output_file_dict['cr_reject_output'][1])
print 'Done running cr_reject'
else:
print 'Skipping cr_reject'
# Run astrodrizzle.
if astrodrizzle_switch:
output_check = check_for_outputs(output_file_dict['drizzle_output'])
if reproc_switch == False and output_check == True:
print 'Not reprocessing astrodrizzle files.'
else:
print 'Running Astrodrizzle'
for filename in output_file_dict['cr_reject_output']:
updatewcs.updatewcs(filename)
run_astrodrizzle(filename)
print 'Done running astrodrizzle'
else:
print 'Skipping astrodrizzle'
# Run trim.
if png_switch:
output_check = check_for_outputs(output_file_dict['png_output'])
if reproc_switch == False and output_check == True:
print 'Not reprocessing png files.'
else:
print 'Running png'
for filename, weight_file in zip(output_file_dict['drizzle_output'], \
output_file_dict['drizzle_weight']):
output_path = os.path.join(os.path.dirname(filename), 'png')
run_trim(filename, weight_file, output_path)
print 'Done running png'
else:
print 'Skipping running png'
def test_fuv_mama(self):
""" This test confirms that drizzlepac can correcly apply the
distortion model for STIS FUV MAMA data and create a combined product.
"""
# Prepare input files.
raw_inputs = [
'o60q02f3q_flt.fits', 'o60q02f4q_flt.fits', 'o60q02f6q_flt.fits',
'o60q02f8q_flt.fits', 'o60q02fbq_flt.fits', 'o60q02fcq_flt.fits',
'o60q02feq_flt.fits', 'o60q02fgq_flt.fits', 'o60q02fiq_flt.fits'
]
all_inputs = [self.get_input_file('input', i) for i in raw_inputs]
inputs = [os.path.basename(i) for i in all_inputs]
print("[STIS_01] Found inputs: \n{}".format(inputs))
output = 'final_stis_01'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_stis_01.fits'
# Update WCS for all inputs
updatewcs.updatewcs(inputs, use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = True
adriz_parobj['runfile'] = 'stis_01'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 1: STATIC MASK']['static_sig'] = 3.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skystat'] = 'mean'
adriz_parobj['STEP 2: SKY SUBTRACTION']['skywidth'] = 0.1
adriz_parobj['STEP 2: SKY SUBTRACTION']['skylower'] = -50.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skyupper'] = 200.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['use_static'] = False
adriz_parobj['STEP 2: SKY SUBTRACTION']['sky_bits'] = None
adriz_parobj['STEP 3: DRIZZLE SEPARATE IMAGES'][
'driz_separate'] = False
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['median'] = False
adriz_parobj['STEP 5: BLOT BACK THE MEDIAN IMAGE']['blot'] = False
adriz_parobj['STEP 6: REMOVE COSMIC RAYS WITH DERIV, DRIZ_CR'][
'driz_cr'] = False
astrodrizzle.AstroDrizzle(inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def test_stis_ccd(self):
""" This test confirms that drizzlepac can correcly apply the
distortion model for STIS CCD data and create a combined product.
"""
# Prepare input files.
raw_inputs = [
'o6cl10arq_flt.fits', 'o6cl10asq_flt.fits', 'o6cl10atq_flt.fits',
'o6cl10auq_flt.fits', 'o6cl10axq_flt.fits', 'o6cl10ayq_flt.fits',
'o6cl10azq_flt.fits', 'o6cl10b0q_flt.fits'
]
all_inputs = [self.get_input_file('input', i) for i in raw_inputs]
inputs = [os.path.basename(i) for i in all_inputs]
output = 'final_stis_04'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_stis_04.fits'
# Update WCS for all inputs
updatewcs.updatewcs(inputs, use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = True
adriz_parobj['runfile'] = 'stis_04'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 1: STATIC MASK']['static_sig'] = 3.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skywidth'] = 0.1
adriz_parobj['STEP 2: SKY SUBTRACTION']['skylower'] = -50.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skyupper'] = 200.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['use_static'] = False
adriz_parobj['STEP 2: SKY SUBTRACTION']['sky_bits'] = None
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['combine_maskpt'] = 0.7
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['combine_type'] = 'median'
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['combine_nsigma'] = '6 3'
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['combine_nhigh'] = 1
adriz_parobj['STEP 7: DRIZZLE FINAL COMBINED IMAGE'][
'final_bits'] = 528
astrodrizzle.AstroDrizzle(inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def test_perf(self, output, in_memory, num_cores, use_static, skywidth):
# Prepare input files.
raw_inputs = ["ib6m02d9q_flt.fits", "ib6m02daq_flt.fits"]
inputs = [os.path.basename(self.get_input_file('input', i))
for i in raw_inputs]
# Merge common parameter settings with test-specific settings
input_pars = {'build':True, 'preserve':False,
'clean':True, 'sky_bits':None}
input_pars['output'] = output
input_pars['in_memory'] = in_memory
input_pars['num_cores'] = num_cores
input_pars['use_static'] = use_static
input_pars['skywidth'] = skywidth
run_file = '{}.log'.format(output)
input_pars['runfile'] = run_file
# Update WCS for all inputs
driz_inputs = updatewcs.updatewcs(inputs, use_db=False)
# run astrodrizzle now...
parObj = teal.load('astrodrizzle', defaults=True) # get all default values
astrodrizzle.AstroDrizzle(driz_inputs, configobj=parObj, **input_pars)
# Compare results
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_{}.fits'.format(output)
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def test_uvis_single(self):
rootname = 'iacr51ohq'
input_name = '{}_flt.fits'.format(rootname)
output = '{}_drz.fits'.format(rootname)
ref_file = 'reference_wfc3_uvis_single.fits'
runfile = ref_file.replace('.fits','.log')
# Prepare input files.
input_file = os.path.basename(self.get_input_file('input', input_name))
input_pars = {'output': output, 'runfile':runfile,
'build':True, 'in_memory':True,
'preserve': False, 'clean':True, 'static':True,
'skysub':True, 'skywidth':0.3, 'use_static':False,
'sky_bits':None,
'driz_separate':False, 'median':False,
'blot':False, 'driz_cr':False}
# Update wcs in input file
driz_input = updatewcs.updatewcs(input_file)
# run astrodrizzle now...
parObj = teal.load('astrodrizzle', defaults=True) # get all default values
astrodrizzle.AstroDrizzle(driz_input, mdriztab=False,
configobj=parObj, **input_pars)
# Compare results
outputs = [(output, ref_file)]
self.compare_outputs(outputs)
def test_mef_asn(self):
""" This test confirms that drizzlepac can correcly process input
WFPC2 data stored in Multi-extensions FITS(MEF) format.
"""
# Prepare input files.
raw_inputs = ['u9yq0703m_c0m.fits', 'u9yq0704m_c0m.fits',
'u9yq0707m_c0m.fits', 'u9yq0708m_c0m.fits',
'u9yq0703m_c1m.fits', 'u9yq0704m_c1m.fits',
'u9yq0707m_c1m.fits', 'u9yq0708m_c1m.fits']
inputs = [os.path.basename(self.get_input_file('input', i))
for i in raw_inputs]
output = 'wfpc2_mef'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_wfpc2_asn_mef.fits'
# Update WCS for all inputs
driz_inputs = updatewcs.updatewcs(inputs[:4], use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = True
adriz_parobj['runfile'] = 'wfpc2_asn_mef.log'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 2: SKY SUBTRACTION']['skysub'] = False
astrodrizzle.AstroDrizzle(driz_inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def test_fuv_mama(self):
""" This test confirms that drizzlepac can correcly apply the
distortion model for STIS FUV MAMA data and create a combined product.
"""
# Prepare input files.
raw_inputs = ['o60q02f3q_flt.fits', 'o60q02f4q_flt.fits',
'o60q02f6q_flt.fits', 'o60q02f8q_flt.fits',
'o60q02fbq_flt.fits', 'o60q02fcq_flt.fits',
'o60q02feq_flt.fits', 'o60q02fgq_flt.fits',
'o60q02fiq_flt.fits']
all_inputs = [self.get_input_file('input', i) for i in raw_inputs]
inputs = [os.path.basename(i) for i in all_inputs]
print("[STIS_01] Found inputs: \n{}".format(inputs))
output = 'final_stis_01'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_stis_01.fits'
# Update WCS for all inputs
updatewcs.updatewcs(inputs, use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = True
adriz_parobj['runfile'] = 'stis_01'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 1: STATIC MASK']['static_sig'] = 3.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skystat'] = 'mean'
adriz_parobj['STEP 2: SKY SUBTRACTION']['skywidth'] = 0.1
adriz_parobj['STEP 2: SKY SUBTRACTION']['skylower'] = -50.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skyupper'] = 200.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['use_static'] = False
adriz_parobj['STEP 2: SKY SUBTRACTION']['sky_bits'] = None
adriz_parobj['STEP 3: DRIZZLE SEPARATE IMAGES']['driz_separate'] = False
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['median'] = False
adriz_parobj['STEP 5: BLOT BACK THE MEDIAN IMAGE']['blot'] = False
adriz_parobj['STEP 6: REMOVE COSMIC RAYS WITH DERIV, DRIZ_CR']['driz_cr'] = False
astrodrizzle.AstroDrizzle(inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def test_stis_ccd(self):
""" This test confirms that drizzlepac can correcly apply the
distortion model for STIS CCD data and create a combined product.
"""
# Prepare input files.
raw_inputs = ['o6cl10arq_flt.fits', 'o6cl10asq_flt.fits',
'o6cl10atq_flt.fits', 'o6cl10auq_flt.fits',
'o6cl10axq_flt.fits', 'o6cl10ayq_flt.fits',
'o6cl10azq_flt.fits', 'o6cl10b0q_flt.fits']
all_inputs = [self.get_input_file('input', i) for i in raw_inputs]
inputs = [os.path.basename(i) for i in all_inputs]
output = 'final_stis_04'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_stis_04.fits'
# Update WCS for all inputs
updatewcs.updatewcs(inputs, use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = True
adriz_parobj['runfile'] = 'stis_04'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 1: STATIC MASK']['static_sig'] = 3.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skywidth'] = 0.1
adriz_parobj['STEP 2: SKY SUBTRACTION']['skylower'] = -50.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skyupper'] = 200.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['use_static'] = False
adriz_parobj['STEP 2: SKY SUBTRACTION']['sky_bits'] = None
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['combine_maskpt'] = 0.7
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['combine_type'] = 'median'
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['combine_nsigma'] = '6 3'
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['combine_nhigh'] = 1
adriz_parobj['STEP 7: DRIZZLE FINAL COMBINED IMAGE']['final_bits'] = 528
astrodrizzle.AstroDrizzle(inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def mytweakreg_quad_main(path, refim):
os.chdir(path)
os.system('pwd')
updatewcs.updatewcs('*flc.fits')
teal.unlearn('tweakreg')
findpars = {
'computesig': True,
'threshold': 100.,
'conv_width': 3.5,
'peakmin': 100.,
'peakmax': 90000.,
'nsigma': 1.5,
'ratio': 1.0,
'theta': 0.0,
'dqbits': None,
'use_sharp_round': False,
'ylimit': 0.1
}
tweakreg.TweakReg(glob.glob('*flc.fits'),
expand_refcat=False,
enforce_user_order=False,
updatehdr=False,
shiftfile=True,
writecat=True,
clean=False,
interactive=True,
verbose=False,
headerlet=False,
minobj=15,
searchrad=250.0,
searchunits='pixels',
use2dhist=True,
see2dplot=True,
separation=0.5,
fitgeometry='rscale',
residplot='both',
labelsize=8,
nclip=5,
sigma=3.0,
refimage=refim,
imagefindcfg=findpars,
refimagefindcfg=findpars)
def upwcs(flt):
try:
hdr = fits.open(flt)[0].header
print hdr['FILTER'], hdr['IMAGETYP']
# if hdr['FILTER'] =='BLANK' or hdr['FILTER'] == 'Blank' or \
# hdr['IMAGETYP'] == 'DARK' or hdr['IMAGETYP']=='BIAS' or hdr['PHOTCORR']== 'OMIT':
if hdr['DRIZCORR'] == 'OMIT':
return
else:
print 'UPDATING WCS of {}'.format(flt)
print '_______________________________________'
updatewcs.updatewcs(flt)
except Exception, e:
print flt
print hdr
# print fits.info(flt)
print e
print traceback.format_exc()
def update_wcs( input_filename, files=None):
'''
PURPOSE : TO UPDATE THE WCS USING THE DRIZZLEPAC
NOT SO EASY AS DIFFERENT AGED OBSERVATIONS APPEARS TO NEED DIFFERENT
UPDATE WCS
'''
#If i am updating various flt files some new some old thne
#if new ones use a different module.
#loop over inidivudla flt not *flt.fits
if files is None:
files = glob.glob(input_filename)
for iFile in files:
updatewcs.updatewcs(iFile)
def run_align(self, input_filenames):
# protect against getting a single exposure filename or rootname as input
if not isinstance(input_filenames, list):
input_filenames = [input_filenames]
self.curdir = os.getcwd()
all_files = []
for infile in input_filenames:
downloaded_files = self.get_input_file(infile, docopy=True)
updatewcs.updatewcs(downloaded_files)
all_files.extend(downloaded_files)
gaia_catalog, shift_file_name = align_to_gaia.align(
all_files, shift_name=self.output_shift_file)
shift_file = Table.read(shift_file_name, format='ascii')
return shift_file, all_files
def test_align_ngc188(self):
""" Verify whether NGC188 exposures can be aligned to an astrometric standard.
Characeteristics of this test:
* Input exposures include both ACS and WFC3 images of the same general field-of-view
of NGC188 suitable for creating a combined mosaic using both instruments.
"""
for infile in self.input_filenames:
self.get_input_file(infile, docopy=True)
updatewcs.updatewcs(infile)
output_shift_file = 'test_mosaic_ngc188_shifts.txt'
align_to_gaia.align(self.input_filenames, shift_name=output_shift_file)
shift_file = Table.read(output_shift_file, format='ascii')
rms_x = max(shift_file['col6'])
rms_y = max(shift_file['col7'])
assert (rms_x <= 0.25 and rms_y <= 0.25)
def _process_input_wcs_single(fname, wcskey, updatewcs):
"""
See docs for _process_input_wcs.
This is separated to be spawned in parallel.
"""
if wcskey in ['', ' ', 'INDEF', None]:
if updatewcs:
uw.updatewcs(fname, checkfiles=False)
else:
numext = fileutil.countExtn(fname)
extlist = []
for extn in range(1, numext + 1):
extlist.append(('SCI', extn))
if wcskey in string.ascii_uppercase:
wkey = wcskey
wname = ' '
else:
wname = wcskey
wkey = ' '
altwcs.restoreWCS(fname, extlist, wcskey=wkey, wcsname=wname)
# make an asn table at the end
# Make sure there is a WCSCORR table for each input image
if wcskey not in ['', ' ', 'INDEF', None] or updatewcs:
wcscorr.init_wcscorr(fname)
def _process_input_wcs_single(fname, wcskey, updatewcs):
"""
See docs for _process_input_wcs.
This is separated to be spawned in parallel.
"""
if wcskey in ['', ' ', 'INDEF', None]:
if updatewcs:
uw.updatewcs(fname, checkfiles=False)
else:
numext = fileutil.countExtn(fname)
extlist = []
for extn in range(1, numext + 1):
extlist.append(('SCI', extn))
if wcskey in string.ascii_uppercase:
wkey = wcskey
wname = ' '
else:
wname = wcskey
wkey = ' '
altwcs.restoreWCS(fname, extlist, wcskey=wkey, wcsname=wname)
# make an asn table at the end
# Make sure there is a WCSCORR table for each input image
if wcskey not in ['', ' ', 'INDEF', None] or updatewcs:
wcscorr.init_wcscorr(fname)
def initImage(fitsfile):
"""Prepare an image for pipeline run
- Checks if ACSWFC NPOL needs updating
- Checks if moving target
- Returns metadata (instdet, filter)
"""
instdet = hutils.getInstDet(fitsfile)
hdr = fits.getheader(fitsfile)
if instdet == 'acswfc':
if not 'SIPNAME' in list(hdr.keys()):
print('Updating NPOL')
updatenpol.update(fitsfile, 'jref$')
updatewcs.updatewcs(fitsfile)
wnames = stwcs.wcsutil.altwcs.wcsnames(fitsfile, 1)
if 'IDCFIX' in wnames.values():
stwcs.wcsutil.altwcs.restoreWCS(fitsfile, [1, 4], wcsname='IDCFIX')
print('Fixing empty keys')
with fits.open(fitsfile, mode='update') as hdu:
for ext in [1, 4]:
tddkeys = [
'TDDALPHA', 'TDDBETA', 'TDD_CTA', 'TDD_CTB', 'TDD_CYA',
'TDD_CYB', 'TDD_CXA', 'TDD_CXB'
]
for k in tddkeys:
if isinstance(hdu[ext].header[k], Undefined):
hdu[ext].header[k] = ''
hdu.flush()
if instdet == 'wfc3ir':
if 'WCSAXESO' not in list(hdr.keys()):
print('Updating WCS')
updatewcs.updatewcs(fitsfile)
wnames = stwcs.wcsutil.altwcs.wcsnames(fitsfile, 1)
if 'IDCFIX' in wnames.values():
stwcs.wcsutil.altwcs.restoreWCS(fitsfile, 1, wcsname='IDCFIX')
if instdet == 'wfc3uvis':
if 'WCSAXESO' not in list(hdr.keys()):
print('Updating WCS')
updatewcs.updatewcs(fitsfile)
wnames = stwcs.wcsutil.altwcs.wcsnames(fitsfile, 1)
if 'IDCFIX' in wnames.values():
stwcs.wcsutil.altwcs.restoreWCS(fitsfile, [1, 4],
wcsname='IDCFIX')
print('Checking if moving target observation')
with fits.open(fitsfile, mode='update') as hdu:
hdr = hdu[0].header
if hdr['mtflag'] == 'T':
hdr['mtflag'] = ' '
filter = hutils.getFilter(fitsfile)
return instdet, filter
def runmakewcs(input):
"""
Runs make wcs and recomputes the WCS keywords
Parameters
----------
input : str or list of str
a list of files
Returns
-------
output : list of str
returns a list of names of the modified files
(For GEIS files returns the translated names.)
"""
newNames = uw.updatewcs(input, checkfiles=False)
#newNames = makewcs.run(input)
return newNames
def runmakewcs(input):
"""
Runs make wcs and recomputes the WCS keywords
Parameters
----------
input : str or list of str
a list of files
Returns
-------
output : list of str
returns a list of names of the modified files
(For GEIS files returns the translated names.)
"""
newNames = uw.updatewcs(input, checkfiles=False)
#newNames = makewcs.run(input)
return newNames
def test_waiver_single(self):
""" This test confirms that drizzlepac can correcly process .
"""
# Prepare input files.
raw_inputs = ["u40x010hm_c0f.fits", "u40x010hm_c1f.fits"]
inputs = [
os.path.basename(self.get_input_file('input', i))
for i in raw_inputs
]
output = 'wfpc2_single_waiver'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_single_waiver.fits'
# Update WCS for all inputs
driz_inputs = updatewcs.updatewcs(inputs[0], use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = False
adriz_parobj['runfile'] = 'wfpc2_single_waiver.log'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 1: STATIC MASK']['static'] = False
adriz_parobj['STEP 2: SKY SUBTRACTION']['skysub'] = True
adriz_parobj['STEP 2: SKY SUBTRACTION']['skystat'] = 'mode'
adriz_parobj['STEP 2: SKY SUBTRACTION']['skywidth'] = 0.3
adriz_parobj['STEP 2: SKY SUBTRACTION']['skylower'] = -100.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['use_static'] = False
adriz_parobj['STEP 2: SKY SUBTRACTION']['sky_bits'] = None
adriz_parobj['STEP 3: DRIZZLE SEPARATE IMAGES'][
'driz_separate'] = False
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['median'] = False
adriz_parobj['STEP 5: BLOT BACK THE MEDIAN IMAGE']['blot'] = False
adriz_parobj['STEP 6: REMOVE COSMIC RAYS WITH DERIV, DRIZ_CR'][
'driz_cr'] = False
astrodrizzle.AstroDrizzle(driz_inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def runmakewcs(input):
"""
Runs 'updatewcs' to recompute the WCS keywords for the input image.
Parameters
----------
input : list of str
A list of file names.
Returns
-------
output : list of str
Returns a list of names of the modified files
(For GEIS files returns the translated names).
"""
from stwcs import updatewcs
newNames = updatewcs.updatewcs(input, checkfiles=False)
return newNames
def test_waiver_single(self):
""" This test confirms that drizzlepac can correcly process .
"""
# Prepare input files.
raw_inputs = ["u40x010hm_c0f.fits", "u40x010hm_c1f.fits"]
inputs = [os.path.basename(self.get_input_file('input', i))
for i in raw_inputs]
output = 'wfpc2_single_waiver'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_single_waiver.fits'
# Update WCS for all inputs
driz_inputs = updatewcs.updatewcs(inputs[0], use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = False
adriz_parobj['runfile'] = 'wfpc2_single_waiver.log'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 1: STATIC MASK']['static'] = False
adriz_parobj['STEP 2: SKY SUBTRACTION']['skysub'] = True
adriz_parobj['STEP 2: SKY SUBTRACTION']['skystat'] = 'mode'
adriz_parobj['STEP 2: SKY SUBTRACTION']['skywidth'] = 0.3
adriz_parobj['STEP 2: SKY SUBTRACTION']['skylower'] = -100.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['use_static'] = False
adriz_parobj['STEP 2: SKY SUBTRACTION']['sky_bits'] = None
adriz_parobj['STEP 3: DRIZZLE SEPARATE IMAGES']['driz_separate'] = False
adriz_parobj['STEP 4: CREATE MEDIAN IMAGE']['median'] = False
adriz_parobj['STEP 5: BLOT BACK THE MEDIAN IMAGE']['blot'] = False
adriz_parobj['STEP 6: REMOVE COSMIC RAYS WITH DERIV, DRIZ_CR']['driz_cr'] = False
astrodrizzle.AstroDrizzle(driz_inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def test_waiver_asn(self):
""" This test confirms that drizzlepac can correcly process input
WFPC2 data stored in WAIVER fits format.
"""
# Prepare input files.
raw_inputs = [
'u40x010hm_c0f.fits', 'u40x010im_c0f.fits', 'u40x010jm_c0f.fits',
'u40x010km_c0f.fits', 'u40x010hm_c1f.fits', 'u40x010im_c1f.fits',
'u40x010jm_c1f.fits', 'u40x010km_c1f.fits'
]
inputs = [
os.path.basename(self.get_input_file('input', i))
for i in raw_inputs
]
output = 'wfpc2_waiver'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_wfpc2_asn_waiver.fits'
# Update WCS for all inputs
driz_inputs = updatewcs.updatewcs(inputs[:4], use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = True
adriz_parobj['runfile'] = 'wfpc2_asn_waiver.log'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 1: STATIC MASK']['static_sig'] = 3.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skysub'] = False
astrodrizzle.AstroDrizzle(driz_inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def test_mef_asn(self):
""" This test confirms that drizzlepac can correcly process input
WFPC2 data stored in Multi-extensions FITS(MEF) format.
"""
# Prepare input files.
raw_inputs = [
'u9yq0703m_c0m.fits', 'u9yq0704m_c0m.fits', 'u9yq0707m_c0m.fits',
'u9yq0708m_c0m.fits', 'u9yq0703m_c1m.fits', 'u9yq0704m_c1m.fits',
'u9yq0707m_c1m.fits', 'u9yq0708m_c1m.fits'
]
inputs = [
os.path.basename(self.get_input_file('input', i))
for i in raw_inputs
]
output = 'wfpc2_mef'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_wfpc2_asn_mef.fits'
# Update WCS for all inputs
driz_inputs = updatewcs.updatewcs(inputs[:4], use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = True
adriz_parobj['runfile'] = 'wfpc2_asn_mef.log'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 2: SKY SUBTRACTION']['skysub'] = False
astrodrizzle.AstroDrizzle(driz_inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def test_waiver_asn(self):
""" This test confirms that drizzlepac can correcly process input
WFPC2 data stored in WAIVER fits format.
"""
# Prepare input files.
raw_inputs = ['u40x010hm_c0f.fits', 'u40x010im_c0f.fits',
'u40x010jm_c0f.fits', 'u40x010km_c0f.fits',
'u40x010hm_c1f.fits', 'u40x010im_c1f.fits',
'u40x010jm_c1f.fits', 'u40x010km_c1f.fits']
inputs = [os.path.basename(self.get_input_file('input', i))
for i in raw_inputs]
output = 'wfpc2_waiver'
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_wfpc2_asn_waiver.fits'
# Update WCS for all inputs
driz_inputs = updatewcs.updatewcs(inputs[:4], use_db=False)
# run astrodrizzle now...
adriz_parobj = teal.load('astrodrizzle', defaults=True)
adriz_parobj['output'] = output
adriz_parobj['build'] = True
adriz_parobj['in_memory'] = True
adriz_parobj['runfile'] = 'wfpc2_asn_waiver.log'
adriz_parobj['STATE OF INPUT FILES']['preserve'] = False
adriz_parobj['STATE OF INPUT FILES']['clean'] = True
adriz_parobj['STEP 1: STATIC MASK']['static_sig'] = 3.0
adriz_parobj['STEP 2: SKY SUBTRACTION']['skysub'] = False
astrodrizzle.AstroDrizzle(driz_inputs, configobj=adriz_parobj)
# Compare results
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def subtract_flt_background(root='GOODN-N1-VBA-F105W',
scattered_light=False,
sex_background=False,
order=2):
"""
Subtract polynomial background
"""
import scipy.optimize
import astropy.units as u
from astropy.table import Table as table
import stwcs
from stwcs import updatewcs
import drizzlepac
from drizzlepac import astrodrizzle, tweakreg, tweakback
import threedhst
asn = threedhst.utils.ASNFile(root + '_asn.fits')
for exp in asn.exposures:
updatewcs.updatewcs('%s_%s.fits' % (exp, 'flt'))
if not os.path.exists('%s_drz_sci.fits' % (root)):
if len(asn.exposures) == 1:
drizzlepac.astrodrizzle.AstroDrizzle(root + '_asn.fits',
clean=False,
context=False,
preserve=False,
skysub=True,
driz_separate=False,
driz_sep_wcs=False,
median=False,
blot=False,
driz_cr=False,
driz_cr_corr=False,
driz_combine=True)
else:
drizzlepac.astrodrizzle.AstroDrizzle(root + '_asn.fits',
clean=False,
context=False,
preserve=False,
skysub=True,
driz_separate=True,
driz_sep_wcs=True,
median=True,
blot=True,
driz_cr=True,
driz_cr_corr=True,
driz_combine=True)
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = '%s_drz_wht.fits' % (root)
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION,BACKGROUND'
se.options['CHECKIMAGE_NAME'] = '%s_drz_seg.fits,%s_drz_bkg.fits' % (root,
root)
se.options['BACK_TYPE'] = 'AUTO'
se.options['BACK_SIZE'] = '256'
#
se.params['X_IMAGE'] = True
se.params['Y_IMAGE'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = '%s_drz_sci.cat' % (root)
se.options['FILTER'] = 'Y'
se.copyConvFile()
se.options['FILTER_NAME'] = 'gauss_4.0_7x7.conv'
se.options['DETECT_THRESH'] = '0.8'
se.options['ANALYSIS_THRESH'] = '0.8'
#
se.options['MEMORY_OBJSTACK'] = '30000'
se.options['MEMORY_PIXSTACK'] = '3000000'
se.options['MEMORY_BUFSIZE'] = '2048'
se.sextractImage('%s_drz_sci.fits' % (root))
#threedhst.sex.sexcatRegions('%s_flt.cat' %(exp), '%s_flt.reg' %(exp), format=1)
#### Blot segmentation map to FLT images for object mask
asn = threedhst.utils.ASNFile('%s_asn.fits' % (root))
#print 'Read files...'
ref = pyfits.open('%s_drz_sci.fits' % (root))
ref_wcs = stwcs.wcsutil.HSTWCS(ref, ext=0)
seg = pyfits.open('%s_drz_seg.fits' % (root))
#### Fill ref[0].data with zeros for seg mask
#seg_data = ref[0].data
#seg_data[seg[0].data == 0] = 0
seg_data = np.cast[np.float32](seg[0].data)
bkg_data = pyfits.open('%s_drz_bkg.fits' % (root))[0].data
yi, xi = np.indices((1014, 1014))
if scattered_light:
bg_components = np.ones((4, 1014, 1014))
bg_components[1, :, :] = xi / 1014. * 2
bg_components[2, :, :] = yi / 1014. * 2
bg_components[3, :, :] = pyfits.open(
os.getenv('THREEDHST') + '/CONF/G141_scattered_light.fits')[0].data
#### Use flat-field itself for images affected by full-field
#### persistence from the tungsten lamp
if scattered_light == 2:
bg_components[3, :, :] = pyfits.open(
os.getenv('iref') + 'flat_UDF_F140W_v0.fits')[1].data[5:-5,
5:-5]
NCOMP = 4
else:
# bg_components = np.ones((3,1014,1014))
# bg_components[1,:,:] = xi/1014.*2
# bg_components[2,:,:] = yi/1014.*2
# NCOMP=3
#
if order == 2:
NCOMP = 6
bg_components = np.ones((NCOMP, 1014, 1014))
bg_components[1, :, :] = (xi - 507) / 507.
bg_components[2, :, :] = (yi - 507) / 507.
bg_components[3, :, :] = ((xi - 507) / 507.)**2
bg_components[4, :, :] = ((yi - 507) / 507.)**2
bg_components[5, :, :] = (xi - 507) * (yi - 507) / 507.**2
else:
NCOMP = 3
bg_components = np.ones((NCOMP, 1014, 1014))
bg_components[1, :, :] = (xi - 507) / 507.
bg_components[2, :, :] = (yi - 507) / 507.
bg_flat = bg_components.reshape((NCOMP, 1014**2))
#### Loop through FLTs, blotting reference and segmentation
models = []
for exp in asn.exposures:
flt = pyfits.open('%s_flt.fits' % (exp)) #, mode='update')
flt_wcs = stwcs.wcsutil.HSTWCS(flt, ext=1)
### segmentation
print('Segmentation image: %s_blot.fits' % (exp))
blotted_seg = astrodrizzle.ablot.do_blot(seg_data + 0,
ref_wcs,
flt_wcs,
1,
coeffs=True,
interp='nearest',
sinscl=1.0,
stepsize=10,
wcsmap=None)
blotted_bkg = 0.
if sex_background:
blotted_bkg = astrodrizzle.ablot.do_blot(bkg_data + 0,
ref_wcs,
flt_wcs,
1,
coeffs=True,
interp='nearest',
sinscl=1.0,
stepsize=10,
wcsmap=None)
flt[1].data -= blotted_bkg
mask = (blotted_seg == 0) & (flt['DQ'].data
== 0) & (flt[1].data > -1) & (xi > 10) & (
yi > 10) & (xi < 1004) & (yi < 1004)
mask &= np.isfinite(flt[1].data) & np.isfinite(flt[2].data)
mask &= (flt[1].data < 5 * np.median(flt[1].data[mask]))
data_range = np.percentile(flt[1].data[mask], [2.5, 97.5])
mask &= (flt[1].data >= data_range[0]) & (flt[1].data <= data_range[1])
data_range = np.percentile(flt[2].data[mask], [0.5, 99.5])
mask &= (flt[2].data >= data_range[0]) & (flt[2].data <= data_range[1])
### Least-sq fit for component normalizations
data = flt[1].data[mask].flatten()
wht = (1. / flt[2].data[mask].flatten())**2
templates = bg_flat[:, mask.flatten()]
p0 = np.zeros(NCOMP)
p0[0] = np.median(data)
obj_fun = threedhst.grism_sky.obj_lstsq
print('XXX: %d' % (mask.sum()))
popt = scipy.optimize.leastsq(obj_fun,
p0,
args=(data, templates, wht),
full_output=True,
ftol=1.49e-8 / 1000.,
xtol=1.49e-8 / 1000.)
xcoeff = popt[0]
model = np.dot(xcoeff, bg_flat).reshape((1014, 1014))
models.append(model)
# add header keywords of the fit components
flt = pyfits.open('%s_flt.fits' % (exp), mode='update')
flt[1].data -= model + blotted_bkg
for i in range(NCOMP):
if 'BGCOMP%d' % (i + 1) in flt[0].header:
flt[0].header['BGCOMP%d' % (i + 1)] += xcoeff[i]
else:
flt[0].header['BGCOMP%d' % (i + 1)] = xcoeff[i]
flt.flush()
coeff_str = ' '.join(['%.4f' % c for c in xcoeff])
threedhst.showMessage('Background subtraction, %s_flt.fits:\n\n %s' %
(exp, coeff_str))
return models
def compare_wcs_alignment(dataset, force=False):
"""Return results from aligning dataset using all available WCS solutions.
This code will ALWAYS make sure the ASTROMETRY_STEP_CONTROL variable
is set to "ON" when running and will reset to the original state when
completed. This insures that the code ALWAYS queries the astrometry
database to apply all avaialable a priori WCS solutions.
Parameters
-----------
dataset : str
Rootname of either a single (un-associated) exposure or an ASN
force : bool
Specify whether or not to overwrite dataset files found locally
with fresh copies retrieved from MAST.
Returns
-------
results : dict
A dictionary whose keys are the WCS's found and fit to GAIA.
Each WCS has entries for:
* imageName - filenames of input exposures included in the fit
* offset_x - offset in X (pixels)
* offset_y - offset in X (pixels)
* rotation - rotation in degrees
* scale - scale from fit
* rms_x - RMS in pixels
* rms_y - RMS in pixels
* fit_rms - RMS in arcseconds
* total_rms - RMS of entire fit in arcseconds
* status - flag indicating success/failure of fit
* fit_qual - flag indicating quality of fit (1-5)
* matched_sources - number of sources used in fit
ASSUMPTIONS
-----------
- All images in dataset have the same set of a priori solutions
- All images in dataset have the same setting for the IDCTAB file
"""
# Setup
# Remember what state the environment was in before this code runs
control = os.environ.get('ASTROMETRY_STEP_CONTROL')
# Insure that database will be queried for new WCS solutions
os.environ['ASTROMETRY_STEP_CONTROL'] = 'ON'
try:
# Step 1:
# Determine alignment for pipeline-defined WCS
align_table = align.perform_align([dataset],
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
clobber=force,
debug=True,
product_type='pipeline')
results = align_table.filtered_table
if not results:
msg = "No valid exposures found for {}.".format(dataset)
msg += "\n Please check that input was either a valid ASN"
msg += "\n or a single un-associated exposure."
raise ValueError(msg)
imglist = results['imageName'].astype(str).tolist()
# Step 2:
# Create results output organized by WCSNAME
default_wcsname = fits.getval(imglist[0], 'wcsname', ext=1)
log.info("Default WCSNAME: {}".format(default_wcsname))
alignment = {default_wcsname: extract_results(results)}
# Download the calibration reference files to ensure availability
ref_files = ref_from_image(
imglist[0], ['IDCTAB', 'DGEOFILE', 'NPOLFILE', 'D2IMFILE'])
for file in ref_files:
download_crds(file, verbose=True)
# Step 3:
# Update inputs with latest distortion model and pull in solutions from dB
imglist = updatewcs.updatewcs(imglist)
img0 = imglist[0]
# Step 4:
# Loop over each WCS solution and perform alignment to GAIA
wcsnames = headerlet.get_headerlet_kw_names(img0, kw='WCSNAME')
if not wcsnames:
msg = "No a priori solutions found for {}".format(img0)
log.error(msg)
raise ValueError(msg)
for wcs in wcsnames:
log.info("Starting with {}".format(wcs))
if 'OPUS' in wcs or wcs == default_wcsname:
continue # skip default pipeline solutions, since we have already aligned it
# apply WCS from headerlet
for img in imglist:
wnames = headerlet.get_headerlet_kw_names(img, kw='WCSNAME')
hnames = headerlet.get_headerlet_kw_names(img)
# print("[testutils]WCSNAMES[{}]: {}".format(img, wnames))
if wcs in wnames:
hdrname = hnames[wnames.index(wcs)]
log.info("[testutils] Applying WCS {} to {}".format(
hdrname, img))
headerlet.restore_from_headerlet(img,
hdrname=hdrname,
archive=False,
force=True)
print("[testutils] Aligning: {} for WCSNAME: {}".format(
dataset, wcs))
align_table = align.perform_align(
[dataset],
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
clobber=False,
debug=True,
product_type='pipeline')
results = align_table.filtered_table
alignment[wcs] = extract_results(results)
except Exception as err:
print(traceback.format_exc())
raise err
finally:
# Regardless of what happens, always reset the environment variable
# if it was modified in the first place.
# Restore user environment to original state
if control is None: # Need to be explicit here since T/F are actually valid
del os.environ['ASTROMETRY_STEP_CONTROL']
else:
os.environ['ASTROMETRY_STEP_CONTROL'] = control
return alignment
def test_pixsky1(self):
"""This test verifies that the coordinate transformation tasks
'pixtopix', 'pixtosky', and 'skytopix' still work as expected.
This test relies on the truth/comparison file from the `acs_tweak` test
to define the output reference WCS for these coordinate transformations.
"""
input_file = self.get_input_file('input', 'j94f05bgq_flt.fits')
fpath, froot = os.path.split(input_file)
refwcs_file = self.get_data('truth', 'reference_tweak.fits')
ref_file1 = 'j94f05bgq_flt_reference.coo'
ref_file2 = 'j94f05bgq_flt_sky_reference.coo'
ref_file3 = 'j94f05bgq_flt_sci1_skyxy_reference.coo'
ref_file4 = 'j94f05bgq_flt_sci2_skyxy_reference.coo'
# Run test...
flt_file = updatewcs.updatewcs(froot)
# create combined results file
out_refxy_file = froot.replace('_flt.fits', '_flt_refxy.coo')
out_sky_file = froot.replace('_flt.fits', '_flt_sky.coo')
inskycat = froot.replace('_flt.fits', '_flt_sky_catalog.coo')
inskycat = self.get_data('input', inskycat)
outpixcat_names = []
outpix = open(out_refxy_file, 'w')
outsky = open(out_sky_file, 'w')
for i in [1, 2]:
outcat = froot.replace('_flt.fits', '_flt_sci%d_refxy.coo' % i)
incat = froot.replace('_flt.fits', '_flt_sci%d_xy_catalog.coo' % i)
incat = self.get_data('input', incat)
pixtopix.tran(froot+'[sci,%d]' % i, '{}[sci,1]'.format(refwcs_file),
direction='forward',
x=None, y=None, coordfile=incat, colnames='',
separator=None, precision=6, output=outcat,
verbose=False)
shutil.copyfileobj(open(outcat, 'r'), outpix)
outskycat = froot.replace('_flt.fits', '_flt_sci%d_sky.coo' % i)
pixtosky.xy2rd(froot+'[sci,%d]' % i, coordfile=incat,
x=None, y=None, colnames='', separator=None,
hms=True, precision=6, output=outskycat,
verbose=False)
shutil.copyfileobj(open(outskycat, 'r'), outsky)
outpixcat = froot.replace('_flt.fits', '_flt_sci%d_skyxy.coo' % i)
skytopix.rd2xy(froot+'[sci,%d]' % i, ra=None, dec=None,
coordfile=inskycat, colnames=None, precision=6,
output=outpixcat, verbose=False)
outpixcat_names.append(outpixcat)
# Close combined results files
outpix.close()
outsky.close()
outputs = [(out_refxy_file, ref_file1), (out_sky_file, ref_file2)]
for oname, rname in zip(outpixcat_names, [ref_file3, ref_file4]):
outputs.append((oname, rname))
self.compare_outputs(outputs)
def process(inFile,force=False,newpath=None, inmemory=False, num_cores=None,
headerlets=True, align_to_gaia=True):
""" Run astrodrizzle on input file/ASN table
using default values for astrodrizzle parameters.
"""
# We only need to import this package if a user run the task
import drizzlepac
from drizzlepac import processInput # used for creating new ASNs for _flc inputs
from stwcs import updatewcs
from drizzlepac import alignimages
# interpret envvar variable, if specified
if envvar_compute_name in os.environ:
val = os.environ[envvar_compute_name].lower()
if val not in envvar_bool_dict:
msg = "ERROR: invalid value for {}.".format(envvar_compute_name)
msg += " \n Valid Values: on, off, yes, no, true, false"
raise ValueError(msg)
align_to_gaia = envvar_bool_dict[val]
if envvar_new_apriori_name in os.environ:
# Reset ASTROMETRY_STEP_CONTROL based on this variable
# This provides backward-compatibility until ASTROMETRY_STEP_CONTROL
# gets removed entirely.
val = os.environ[envvar_new_apriori_name].lower()
if val not in envvar_dict:
msg = "ERROR: invalid value for {}.".format(envvar_new_apriori_name)
msg += " \n Valid Values: on, off, yes, no, true, false"
raise ValueError(msg)
os.environ[envvar_old_apriori_name] = envvar_dict[val]
if headerlets or align_to_gaia:
from stwcs.wcsutil import headerlet
# Open the input file
try:
# Make sure given filename is complete and exists...
inFilename = fileutil.buildRootname(inFile,ext=['.fits'])
if not os.path.exists(inFilename):
print("ERROR: Input file - %s - does not exist." % inFilename)
return
except TypeError:
print("ERROR: Inappropriate input file.")
return
#If newpath was specified, move all files to that directory for processing
if newpath:
orig_processing_dir = os.getcwd()
new_processing_dir = _createWorkingDir(newpath,inFilename)
_copyToNewWorkingDir(new_processing_dir,inFilename)
os.chdir(new_processing_dir)
# Initialize for later use...
_mname = None
_new_asn = None
_calfiles = []
# Identify WFPC2 inputs to account for differences in WFPC2 inputs
wfpc2_input = fits.getval(inFilename, 'instrume') == 'WFPC2'
cal_ext = None
# Check input file to see if [DRIZ/DITH]CORR is set to PERFORM
if '_asn' in inFilename:
# We are working with an ASN table.
# Use asnutil code to extract filename
inFilename = _lowerAsn(inFilename)
_new_asn = [inFilename]
_asndict = asnutil.readASNTable(inFilename,None,prodonly=False)
_cal_prodname = _asndict['output'].lower()
#_fname = fileutil.buildRootname(_cal_prodname,ext=['_drz.fits'])
# Retrieve the first member's rootname for possible use later
_fimg = fits.open(inFilename, memmap=False)
for name in _fimg[1].data.field('MEMNAME'):
if name[-1] != '*':
_mname = name.split('\0', 1)[0].lower()
break
_fimg.close()
del _fimg
else:
# Check to see if input is a _RAW file
# If it is, strip off the _raw.fits extension...
_indx = inFilename.find('_raw')
if _indx < 0: _indx = len(inFilename)
# ... and build the CALXXX product rootname.
if wfpc2_input:
# force code to define _c0m file as calibrated product to be used
cal_ext = ['_c0m.fits']
_mname = fileutil.buildRootname(inFilename[:_indx], ext=cal_ext)
_cal_prodname = inFilename[:_indx]
# Reset inFilename to correspond to appropriate input for
# drizzle: calibrated product name.
inFilename = _mname
if _mname is None:
errorMsg = 'Could not find calibrated product!'
raise Exception(errorMsg)
# Create trailer filenames based on ASN output filename or
# on input name for single exposures
if '_raw' in inFile:
# Output trailer file to RAW file's trailer
_trlroot = inFile[:inFile.find('_raw')]
elif '_asn' in inFile:
# Output trailer file to ASN file's trailer, not product's trailer
_trlroot = inFile[:inFile.find('_asn')]
else:
# Default: trim off last suffix of input filename
# and replacing with .tra
_indx = inFile.rfind('_')
if _indx > 0:
_trlroot = inFile[:_indx]
else:
_trlroot = inFile
_trlfile = _trlroot + '.tra'
# Open product and read keyword value
# Check to see if product already exists...
dkey = 'DRIZCORR'
# ...if product does NOT exist, interrogate input file
# to find out whether 'dcorr' has been set to PERFORM
# Check if user wants to process again regardless of DRIZCORR keyword value
if force:
dcorr = 'PERFORM'
else:
if _mname :
_fimg = fits.open(fileutil.buildRootname(_mname,ext=['_raw.fits']), memmap=False)
_phdr = _fimg['PRIMARY'].header
if dkey in _phdr:
dcorr = _phdr[dkey]
else:
dcorr = None
_fimg.close()
del _fimg
else:
dcorr = None
time_str = _getTime()
_tmptrl = _trlroot + '_tmp.tra'
_drizfile = _trlroot + '_pydriz'
_drizlog = _drizfile + ".log" # the '.log' gets added automatically by astrodrizzle
_alignlog = _trlroot + '_align.log'
if dcorr == 'PERFORM':
if '_asn.fits' not in inFilename:
# Working with a singleton
# However, we always want to make sure we always use
# a calibrated product as input, if available.
_infile = fileutil.buildRootname(_cal_prodname, ext=cal_ext)
_infile_flc = fileutil.buildRootname(_cal_prodname,ext=['_flc.fits'])
_cal_prodname = _infile
_inlist = _calfiles = [_infile]
# Add CTE corrected filename as additional input if present
if os.path.exists(_infile_flc) and _infile_flc != _infile:
_inlist.append(_infile_flc)
else:
# Working with an ASN table...
_infile = inFilename
flist,duplist = processInput.checkForDuplicateInputs(_asndict['order'])
_calfiles = flist
if len(duplist) > 0:
origasn = processInput.changeSuffixinASN(inFilename,'flt')
dupasn = processInput.changeSuffixinASN(inFilename,'flc')
_inlist = [origasn,dupasn]
else:
_inlist = [_infile]
# We want to keep the original specification of the calibration
# product name, though, not a lower-case version...
_cal_prodname = inFilename
_new_asn.extend(_inlist) # kept so we can delete it when finished
# check to see whether FLC files are also present, and need to be updated
# generate list of FLC files
align_files = None
_calfiles_flc = [f.replace('_flt.fits','_flc.fits') for f in _calfiles]
# insure these files exist, if not, blank them out
# Also pick out what files will be used for additional alignment to GAIA
if not os.path.exists(_calfiles_flc[0]):
_calfiles_flc = None
align_files = _calfiles
align_update_files = None
else:
align_files = _calfiles_flc
align_update_files = _calfiles
# Run updatewcs on each list of images
updatewcs.updatewcs(_calfiles)
if _calfiles_flc:
updatewcs.updatewcs(_calfiles_flc)
if align_to_gaia:
# Perform additional alignment on the FLC files, if present
###############
#
# call hlapipeline code here on align_files list of files
#
###############
# Create trailer marker message for start of align_to_GAIA processing
_trlmsg = _timestamp("Align_to_GAIA started ")
print(_trlmsg)
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
_trlmsg = ""
# Create an empty astropy table so it can be used as input/output for the perform_align function
#align_table = Table()
try:
align_table = alignimages.perform_align(align_files,update_hdr_wcs=True, runfile=_alignlog)
for row in align_table:
if row['status'] == 0:
trlstr = "Successfully aligned {} to {} astrometric frame\n"
_trlmsg += trlstr.format(row['imageName'], row['catalog'])
else:
trlstr = "Could not align {} to absolute astrometric frame\n"
_trlmsg += trlstr.format(row['imageName'])
except Exception:
# Something went wrong with alignment to GAIA, so report this in
# trailer file
_trlmsg = "EXCEPTION encountered in alignimages...\n"
_trlmsg += " No correction to absolute astrometric frame applied!\n"
# Write the perform_align log to the trailer file...(this will delete the _alignlog)
_appendTrlFile(_trlfile,_alignlog)
# Append messages from this calling routine post-perform_align
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
_trlmsg = ""
#Check to see whether there are any additional input files that need to
# be aligned (namely, FLT images)
if align_update_files and align_table:
# Apply headerlets from alignment to FLT version of the files
for fltfile, flcfile in zip(align_update_files, align_files):
row = align_table[align_table['imageName']==flcfile]
headerletFile = row['headerletFile'][0]
if headerletFile != "None":
headerlet.apply_headerlet_as_primary(fltfile, headerletFile,
attach=True, archive=True)
# append log file contents to _trlmsg for inclusion in trailer file
_trlstr = "Applying headerlet {} as Primary WCS to {}\n"
_trlmsg += _trlstr.format(headerletFile, fltfile)
else:
_trlmsg += "No absolute astrometric headerlet applied to {}\n".format(fltfile)
# Finally, append any further messages associated with alignement from this calling routine
_trlmsg += _timestamp('Align_to_GAIA completed ')
print(_trlmsg)
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
# Run astrodrizzle and send its processing statements to _trlfile
_pyver = drizzlepac.astrodrizzle.__version__
for _infile in _inlist: # Run astrodrizzle for all inputs
# Create trailer marker message for start of astrodrizzle processing
_trlmsg = _timestamp('astrodrizzle started ')
_trlmsg += __trlmarker__
_trlmsg += '%s: Processing %s with astrodrizzle Version %s\n' % (time_str,_infile,_pyver)
print(_trlmsg)
# Write out trailer comments to trailer file...
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
_pyd_err = _trlroot+'_pydriz.stderr'
try:
b = drizzlepac.astrodrizzle.AstroDrizzle(input=_infile,runfile=_drizfile,
configobj='defaults',in_memory=inmemory,
num_cores=num_cores, **pipeline_pars)
except Exception as errorobj:
_appendTrlFile(_trlfile,_drizlog)
_appendTrlFile(_trlfile,_pyd_err)
_ftrl = open(_trlfile,'a')
_ftrl.write('ERROR: Could not complete astrodrizzle processing of %s.\n' % _infile)
_ftrl.write(str(sys.exc_info()[0])+': ')
_ftrl.writelines(str(errorobj))
_ftrl.write('\n')
_ftrl.close()
print('ERROR: Could not complete astrodrizzle processing of %s.' % _infile)
raise Exception(str(errorobj))
# Now, append comments created by PyDrizzle to CALXXX trailer file
print('Updating trailer file %s with astrodrizzle comments.' % _trlfile)
_appendTrlFile(_trlfile,_drizlog)
# Save this for when astropy.io.fits can modify a file 'in-place'
# Update calibration switch
_fimg = fits.open(_cal_prodname, mode='update', memmap=False)
_fimg['PRIMARY'].header[dkey] = 'COMPLETE'
_fimg.close()
del _fimg
# Enforce pipeline convention of all lower-case product
# names
_prodlist = glob.glob('*drz.fits')
for _prodname in _prodlist:
_plower = _prodname.lower()
if _prodname != _plower: os.rename(_prodname,_plower)
else:
# Create default trailer file messages when astrodrizzle is not
# run on a file. This will typically apply only to BIAS,DARK
# and other reference images.
# Start by building up the message...
_trlmsg = _timestamp('astrodrizzle skipped ')
_trlmsg = _trlmsg + __trlmarker__
_trlmsg = _trlmsg + '%s: astrodrizzle processing not requested for %s.\n' % (time_str,inFilename)
_trlmsg = _trlmsg + ' astrodrizzle will not be run at this time.\n'
print(_trlmsg)
# Write message out to temp file and append it to full trailer file
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
# Append final timestamp to trailer file...
_final_msg = '%s: Finished processing %s \n' % (time_str,inFilename)
_final_msg += _timestamp('astrodrizzle completed ')
_trlmsg += _final_msg
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
# If we created a new ASN table, we need to remove it
if _new_asn is not None:
for _name in _new_asn: fileutil.removeFile(_name)
# Clean up any generated OrIg_files directory
if os.path.exists("OrIg_files"):
# check to see whether this directory is empty
flist = glob.glob('OrIg_files/*.fits')
if len(flist) == 0:
os.rmdir("OrIg_files")
else:
print('OrIg_files directory NOT removed as it still contained images...')
# If headerlets have already been written out by alignment code,
# do NOT write out this version of the headerlets
if headerlets:
# Generate headerlets for each updated FLT image
hlet_msg = _timestamp("Writing Headerlets started")
for fname in _calfiles:
frootname = fileutil.buildNewRootname(fname)
hname = "%s_flt_hlet.fits"%frootname
# Write out headerlet file used by astrodrizzle, however,
# do not overwrite any that was already written out by alignimages
if not os.path.exists(hname):
hlet_msg += "Created Headerlet file %s \n"%hname
try:
headerlet.write_headerlet(fname,'OPUS',output='flt', wcskey='PRIMARY',
author="OPUS",descrip="Default WCS from Pipeline Calibration",
attach=False,clobber=True,logging=False)
except ValueError:
hlet_msg += _timestamp("SKIPPED: Headerlet not created for %s \n"%fname)
# update trailer file to log creation of headerlet files
hlet_msg += _timestamp("Writing Headerlets completed")
ftrl = open(_trlfile,'a')
ftrl.write(hlet_msg)
ftrl.close()
# If processing was done in a temp working dir, restore results to original
# processing directory, return to original working dir and remove temp dir
if newpath:
_restoreResults(new_processing_dir,orig_processing_dir)
os.chdir(orig_processing_dir)
_removeWorkingDir(new_processing_dir)
# Provide feedback to user
print(_final_msg)
def compare_sub_to_full_sci(subarray,full_sci,output=False,update=True):
from stsci.tools import fileutil
from stwcs import updatewcs
if update:
# update input SCI file to be consistent with reference files in header
print('Updating input file ',subarray,' to be consistent with reference files listed in header...')
updatewcs.updatewcs(subarray)
print('Updating input file ',full_sci,' to be consistent with reference files listed in header...')
updatewcs.updatewcs(full_sci)
fulldgeofile = fileutil.osfn(pyfits.getval(subarray,'ODGEOFIL'))
# parse out rootname from input file if user wants results written to file
if output:
soutroot = fileutil.buildNewRootname(subarray)
foutroot = fileutil.buildNewRootname(full_sci)
hdulist = pyfits.open(fulldgeofile)
detector = pyfits.getval(fulldgeofile,'DETECTOR')
filter_names = fileutil.getFilterNames(pyfits.getheader(subarray))
# count the number of chips in subarray image
xyfile = pyfits.open(subarray)
numchips = 0
ccdchip = []
extname = xyfile[1].header['EXTNAME']
for extn in xyfile:
if 'extname' in extn.header and extn.header['extname'] == extname:
numchips += 1
if 'ccdchip' in extn.header:
ccdchip.append([extn.header['ccdchip'],extn.header['extver']])
else:
ccdchip.append([1,1])
snx = xyfile['sci',1].header['NAXIS1']
sny = xyfile['sci',1].header['NAXIS2']
ltv1 = xyfile['sci',1].header['ltv1']
ltv2 = xyfile['sci',1].header['ltv2']
xyfile.close()
# build grid of points for full-size image for
# chips corresponding to subarray
xyfile = pyfits.open(full_sci)
fullchip = []
for extn in xyfile:
if ('extname' in extn.header and extn.header['extname'] == extname) and \
extn.header['ccdchip'] == ccdchip[0][0]:
fullchip.append([extn.header['ccdchip'],extn.header['extver']])
xyfile.close()
sxarr,syarr = build_grid_arrays(snx,sny,1)
full_range = [slice(-ltv2,-ltv2+sny),slice(-ltv1,-ltv1+snx)]
fnx = pyfits.getval(full_sci,'NAXIS1','sci',1)
fny = pyfits.getval(full_sci,'NAXIS2','sci',1)
fxarr,fyarr = build_grid_arrays(fnx,fny,1)
# initialize plot here
if has_matplotlib:
pl.clf()
pl.gray()
for chip,det,fext in zip(list(range(1,numchips+1)),ccdchip,fullchip):
# Compute the correction imposed by the D2IM+DGEO corrections
# on the subarray
sxout,syout = transform_d2im_dgeo(subarray,det[1],sxarr,syarr)
sdx= (sxout-sxarr).reshape(sny,snx)
sdy= (syout-syarr).reshape(sny,snx)
# Compute the correction imposed by the D2IM+DGEO corrections
# on the full sized SCI image
fxout,fyout = transform_d2im_dgeo(full_sci,fext[1],fxarr,fyarr)
fdx= (fxout-fxarr).reshape(fny,fnx)
fdy= (fyout-fyarr).reshape(fny,fnx)
# determine the difference
diffx = (sdx - fdx[full_range[0],full_range[1]]).astype(np.float32)
if has_matplotlib:
pl.imshow(diffx)
pl.title('sub_dx-full_x: %s %s[%d:%d,%d:%d] with %g +/- %g' %
(filter_names, detector, full_range[0].start,
full_range[0].stop, full_range[1].start,
full_range[1].stop, diffx.mean(),diffx.std()))
pl.colorbar()
if sys.version_info[0] < 3:
raw_input("Press 'ENTER' to close figure and plot DY...")
else:
input("Press 'ENTER' to close figure and plot DY...")
pl.close()
# determine the difference
diffy = (sdy - fdy[full_range[0],full_range[1]]).astype(np.float32)
if has_matplotlib:
pl.imshow(diffy)
pl.title('sub_dy-full_y: %s %s[%d:%d,%d:%d] with %g +/- %g' %
(filter_names, detector, full_range[0].start,
full_range[0].stop, full_range[1].start,
full_range[1].stop, diffy.mean(), diffy.std()))
pl.colorbar()
if sys.version_info[0] < 3:
raw_input("Press 'ENTER' to close figure and exit...")
else:
input("Press 'ENTER' to close figure and exit...")
pl.close()
if output:
outname = foutroot+'_sci'+str(chip)+'_newfull_dxy.fits'
if os.path.exists(outname): os.remove(outname)
hdulist['dx',chip].data = fdx
hdulist['dy',chip].data = fdy
hdulist.writeto(outname)
outname = soutroot+'_sci'+str(chip)+'_newsub_dxy.fits'
if os.path.exists(outname): os.remove(outname)
hdulist['dx',chip].data = sdx
hdulist['dy',chip].data = sdy
hdulist.writeto(outname)
"""
outname = outroot+'_sci'+str(chip)+'_diff_dxy.fits'
if os.path.exists(outname): os.remove(outname)
hdulist['dx',chip].data = diffx
hdulist['dy',chip].data = diffy
hdulist.writeto(outname)
"""
print('Created output file with differences named: ',outname)
if output:
hdulist.close()
def update(input,refdir="jref$",local=None,interactive=False,wcsupdate=True):
"""
Updates headers of files given as input to point to the new reference files
NPOLFILE and D2IMFILE required with the new C version of MultiDrizzle.
Parameters
-----------
input : string or list
Name of input file or files acceptable forms:
- single filename with or without directory
- @-file
- association table
- python list of filenames
- wildcard specification of filenames
refdir : string
Path to directory containing new reference files, either
environment variable or full path.
local : boolean
Specifies whether or not to copy new reference files to local
directory for use with the input files.
interactive : boolean
Specifies whether or not to interactively ask the user for the
exact names of the new reference files instead of automatically
searching a directory for them.
updatewcs : boolean
Specifies whether or not to update the WCS information in this
file to use the new reference files.
Examples
--------
1. A set of associated images specified by an ASN file can be updated to use
the NPOLFILEs and D2IMFILE found in the local directory defined using
the `myjref$` environment variable under PyRAF using::
>>>import updatenpol
>>>updatenpol.update('j8bt06010_asn.fits', 'myref$')
2. Another use under Python would be to feed it a specific list of files
to be updated using::
>>> updatenpol.update(['file1_flt.fits','file2_flt.fits'],'myjref$')
3. Files in another directory can also be processed using::
>>> updatenpol.update('data$*flt.fits','../new/ref/')
Notes
-----
.. warning::
This program requires access to the `jref$` directory in order
to evaluate the DGEOFILE specified in the input image header.
This evaluation allows the program to get the information it
needs to identify the correct NPOLFILE.
The use of this program now requires that a directory be set up with
all the new NPOLFILE and D2IMFILE reference files for ACS (a single
directory for all files for all ACS detectors will be fine, much like
jref). Currently, all the files generated by the ACS team has initially
been made available at::
/grp/hst/acs/lucas/new-npl/
The one known limitation to how this program works comes from
confusion if more than 1 file could possibly be used as the new
reference file. This would only happen when NPOLFILE reference files
have been checked into CDBS multiple times, and there are several
versions that apply to the same detector/filter combination. However,
that can be sorted out later if we get into that situation at all.
"""
print('UPDATENPOL Version',__version__+'('+__vdate__+')')
# expand (as needed) the list of input files
files,fcol = parseinput.parseinput(input)
if not interactive:
# expand reference directory name (if necessary) to
# interpret IRAF or environment variable names
rdir = fu.osfn(refdir)
ngeofiles,ngcol = parseinput.parseinput(os.path.join(rdir,'*npl.fits'))
# Find D2IMFILE in refdir for updating input file header as well
d2ifiles,d2col = parseinput.parseinput(os.path.join(rdir,"*d2i.fits"))
# Now, build a matched list of input files and DGEOFILE reference files
# to use for selecting the appropriate new reference file from the
# refdir directory.
for f in files:
print('Updating: ',f)
fdir = os.path.split(f)[0]
# Open each file...
fimg = fits.open(f, mode='update')
phdr = fimg['PRIMARY'].header
fdet = phdr['detector']
# get header of DGEOFILE
dfile = phdr.get('DGEOFILE','')
if dfile in ['N/A','',' ',None]:
npolname = ''
else:
dhdr = fits.getheader(fu.osfn(dfile))
if not interactive:
# search all new NPOLFILEs for one that matches current DGEOFILE config
npol = find_npolfile(ngeofiles,fdet,[phdr['filter1'],phdr['filter2']])
else:
if sys.version_info[0] >= 3:
npol = input("Enter name of NPOLFILE for %s:"%f)
else:
npol = raw_input("Enter name of NPOLFILE for %s:"%f)
if npol == "": npol = None
if npol is None:
errstr = "No valid NPOLFILE found in "+rdir+" for detector="+fdet+"\n"
errstr += " filters = "+phdr['filter1']+","+phdr['filter2']
raise ValueError(errstr)
npolname = os.path.split(npol)[1]
if local:
npolname = os.path.join(fdir,npolname)
# clobber any previous copies of this reference file
if os.path.exists(npolname): os.remove(npolname)
shutil.copy(npol,npolname)
else:
if '$' in refdir:
npolname = refdir+npolname
else:
npolname = os.path.join(refdir,npolname)
phdr.set('NPOLFILE', value=npolname,
comment="Non-polynomial corrections in Paper IV LUT",
after='DGEOFILE')
# Now find correct D2IFILE
if not interactive:
d2i = find_d2ifile(d2ifiles,fdet)
else:
if sys.version_info[0] >= 3:
d2i = input("Enter name of D2IMFILE for %s:"%f)
else:
d2i = raw_input("Enter name of D2IMFILE for %s:"%f)
if d2i == "": d2i = None
if d2i is None:
print('=============\nWARNING:')
print(" No valid D2IMFILE found in "+rdir+" for detector ="+fdet)
print(" D2IMFILE correction will not be applied.")
print('=============\n')
d2iname = ""
else:
d2iname = os.path.split(d2i)[1]
if local:
# Copy D2IMFILE to local data directory alongside input file as well
d2iname = os.path.join(fdir,d2iname)
# clobber any previous copies of this reference file
if os.path.exists(d2iname): os.remove(d2iname)
shutil.copy(d2i,d2iname)
else:
if '$' in refdir:
d2iname = refdir+d2iname
else:
d2iname = os.path.join(refdir,d2iname)
phdr.set('D2IMFILE', value=d2iname,
comment="Column correction table",
after='DGEOFILE')
# Close this input file header and go on to the next
fimg.close()
if wcsupdate:
updatewcs.updatewcs(f)
def prep_direct_grism_pair(direct_asn='goodss-34-F140W_asn.fits', grism_asn='goodss-34-G141_asn.fits', radec=None, raw_path='../RAW/', mask_grow=18, scattered_light=False, final_scale=None, skip_direct=False, ACS=False, jump=False, order=2, get_shift=True, align_threshold=20, column_average=True, sky_iter=3, run_acs_lacosmic=False):
"""
Process both the direct and grism observations of a given visit
"""
import threedhst.prep_flt_astrodrizzle as prep
import drizzlepac
from stwcs import updatewcs
import time
t0 = time.time()
#direct_asn='goodss-34-F140W_asn.fits'; grism_asn='goodss-34-G141_asn.fits'; radec=None; raw_path='../RAW/'
#radec = os.getenv('THREEDHST') + '/ASTRODRIZZLE_FLT/Catalog/goodss_radec.dat'
################################
#### Direct image processing
################################
#### xx add astroquery 2MASS/SDSS workaround for radec=None
if not skip_direct:
#### Get fresh FLTS from ../RAW/
asn = threedhst.utils.ASNFile(direct_asn)
if ACS:
for exp in asn.exposures:
print 'cp %s/%s_flc.fits.gz .' %(raw_path, exp)
os.system('cp %s/%s_flc.fits.gz .' %(raw_path, exp))
os.system('gunzip -f %s_flc.fits.gz' %(exp))
if run_acs_lacosmic:
try:
import lacosmicx
status = True
except:
print 'import lacosmicx failed!'
status = False
if status:
im = pyfits.open('%s_flc.fits' %(exp), mode='update')
for ext in [1,2]:
indata = im['SCI',ext].data
#inmask = im['DQ',ext].data > 0
if im['SCI',ext].header['BUNIT'] == 'ELECTRONS':
gain = 1
else:
gain = 1./im[0].header['EXPTIME']
if 'MDRIZSK0' in im['SCI',ext].header:
pssl = im['SCI',ext].header['MDRIZSK0']
else:
pssl = 0.
if 'FLASHLVL' in im[0].header:
pssl += im[0].header['FLASHLVL']
sig_scale = 1.8
else:
sig_scale = 1.
out = lacosmicx.lacosmicx(indata, inmask=None,
sigclip=3.5*sig_scale, sigfrac=0.2,
objlim=7.0, gain=gain,
readnoise=im[0].header['READNSEA'],
satlevel=np.inf, pssl=pssl, niter=5,
sepmed=True, cleantype='meanmask',
fsmode='median', psfmodel='gauss',
psffwhm=2.5,psfsize=7, psfk=None,
psfbeta=4.765, verbose=True)
crmask, cleanarr = out
im['DQ',ext].data |= 16*crmask
### Low pixels
if im[0].header['INSTRUME'] == 'WFC3':
bad = im['SCI',ext].data < -4*im['ERR',ext].data
im['DQ',ext].data |= 16*bad
im.flush()
else:
threedhst.process_grism.fresh_flt_files(direct_asn, from_path=raw_path)
if (not ACS):
#### Subtract WFC3/IR direct backgrounds
prep.subtract_flt_background(root=direct_asn.split('_asn')[0], scattered_light=scattered_light, order=order)
#### Flag IR CRs again within runTweakReg
#### Run TweakReg
if (radec is None) & (not ACS):
print len(asn.exposures)
if len(asn.exposures) > 1:
drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=0.8, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7')
else:
drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=1, context=False, final_bits=576, preserve=False, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_cr_corr=False, driz_combine=True)
else:
if get_shift:
prep.runTweakReg(asn_file=direct_asn, master_catalog=radec, final_scale=None, ACS=ACS, threshold=align_threshold)
#### Subtract background of direct ACS images
if ACS:
for exp in asn.exposures:
flc = pyfits.open('%s_flc.fits' %(exp), mode='update')
if 'SUB' in flc[0].header['APERTURE']:
extensions = [1]
else:
extensions = [1,4]
for ext in extensions:
threedhst.showMessage('Subtract background from %s_flc.fits[%d] : %.4f' %(exp, ext, flc[ext].header['MDRIZSKY']))
flc[ext].data -= flc[ext].header['MDRIZSKY']
flc[ext].header['MDRIZSK0'] = flc[ext].header['MDRIZSKY']
flc[ext].header['MDRIZSKY'] = 0.
#
flc.flush()
else:
pass
#### Do this later, gives segfaults here???
#prep.subtract_flt_background(root=direct_asn.split('_asn')[0], scattered_light=scattered_light)
#### Flag CRs again on BG-subtracted image
#drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=0.8, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7') # ,
################################
#### Grism image processing
################################
if grism_asn:
asn = threedhst.utils.ASNFile(grism_asn)
if ACS:
for exp in asn.exposures:
print 'cp %s/%s_flc.fits.gz .' %(raw_path, exp)
os.system('cp %s/%s_flc.fits.gz .' %(raw_path, exp))
os.system('gunzip -f %s_flc.fits.gz' %(exp))
updatewcs.updatewcs('%s_flc.fits' %(exp))
prep.copy_adriz_headerlets(direct_asn=direct_asn, grism_asn=grism_asn, ACS=True)
prep.subtract_acs_grism_background(asn_file=grism_asn, final_scale=None)
else:
#### Remove the sky and flag CRs
## with mask from rough zodi-only subtraction
prep.subtract_grism_background(asn_file=grism_asn, PATH_TO_RAW='../RAW/', final_scale=None, visit_sky=True, column_average=False, mask_grow=mask_grow, first_run=True)
## Redo making mask from better combined image
prep.subtract_grism_background(asn_file=grism_asn, PATH_TO_RAW='../RAW/', final_scale=final_scale, visit_sky=True, column_average=column_average, mask_grow=mask_grow, first_run=False, sky_iter=sky_iter)
#### Copy headers from direct images
if radec is not None:
prep.copy_adriz_headerlets(direct_asn=direct_asn, grism_asn=grism_asn, ACS=False)
#### Run CR rejection with final shifts
drizzlepac.astrodrizzle.AstroDrizzle(grism_asn, clean=True, skysub=False, final_wcs=True, final_scale=final_scale, final_pixfrac=0.8, context=False, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale='2.5 0.7') # driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7')
if not grism_asn:
t1 = time.time()
threedhst.showMessage('direct: %s\n\nDone (%d s).' %(direct_asn, int(t1-t0)))
else:
t1 = time.time()
threedhst.showMessage('direct: %s\ngrism: %s\n\nDone (%d s).' %(direct_asn, grism_asn, int(t1-t0)))
def openImage(filename, mode='readonly', memmap=0, writefits=True,
clobber=True, fitsname=None):
"""
Opens file and returns PyFITS object. Works on both FITS and GEIS
formatted images.
Notes
-----
If a GEIS or waivered FITS image is used as input, it will convert it to a
MEF object and only if ``writefits = True`` will write it out to a file. If
``fitsname = None``, the name used to write out the new MEF file will be
created using `buildFITSName`.
Parameters
----------
filename: str
name of input file
mode: str
mode for opening file based on PyFITS `mode` parameter values
memmap: int
switch for using memory mapping, 0 for no, 1 for yes
writefits: bool
if True, will write out GEIS as multi-extension FITS
and return handle to that opened GEIS-derived MEF file
clobber: bool
overwrite previously written out GEIS-derived MEF file
fitsname: str
name to use for GEIS-derived MEF file,
if None and writefits==True, will use 'buildFITSName()' to generate one
"""
from stwcs import updatewcs
# Insure that the filename is always fully expanded
# This will not affect filenames without paths or
# filenames specified with extensions.
filename = osfn(filename)
# Extract the rootname and extension specification
# from input image name
_fname, _iextn = parseFilename(filename)
# Check whether we have a FITS file and if so what type
isfits, fitstype = isFits(_fname)
if isfits:
if fitstype != 'waiver':
# Open the FITS file
fimg = fits.open(_fname, mode=mode, memmap=memmap)
return fimg
else:
fimg = convertwaiveredfits.convertwaiveredfits(_fname)
#check for the existence of a data quality file
_dqname = buildNewRootname(_fname, extn='_c1f.fits')
dqexists = os.path.exists(_dqname)
if dqexists:
try:
dqfile = convertwaiveredfits.convertwaiveredfits(_dqname)
dqfitsname = buildNewRootname(_dqname, extn='_c1h.fits')
except:
print("Could not read data quality file %s" % _dqname)
if writefits:
# User wants to make a FITS copy and update it
# using the filename they have provided
if fitsname is None:
rname = buildNewRootname(_fname)
fitsname = buildNewRootname(rname, extn='_c0h.fits')
# Write out GEIS image as multi-extension FITS.
fexists = os.path.exists(fitsname)
if (fexists and clobber) or not fexists:
print('Writing out WAIVERED as MEF to ', fitsname)
fimg.writeto(fitsname, clobber=clobber)
if dqexists:
print('Writing out WAIVERED as MEF to ', dqfitsname)
dqfile.writeto(dqfitsname, clobber=clobber)
# Now close input GEIS image, and open writable
# handle to output FITS image instead...
fimg.close()
del fimg
# Image re-written as MEF, now it needs its WCS updated
updatewcs.updatewcs(fitsname)
fimg = fits.open(fitsname, mode=mode, memmap=memmap)
# Return handle for use by user
return fimg
else:
# Input was specified as a GEIS image, but no FITS copy
# exists. Read it in with 'readgeis' and make a copy
# then open the FITS copy...
try:
# Open as a GEIS image for reading only
fimg = readgeis.readgeis(_fname)
except:
raise IOError("Could not open GEIS input: %s" % _fname)
#check for the existence of a data quality file
_dqname = buildNewRootname(_fname, extn='.c1h')
dqexists = os.path.exists(_dqname)
if dqexists:
try:
dqfile = readgeis.readgeis(_dqname)
dqfitsname = buildFITSName(_dqname)
except:
print("Could not read data quality file %s" % _dqname)
# Check to see if user wanted to update GEIS header.
# or write out a multi-extension FITS file and return a handle to it
if writefits:
# User wants to make a FITS copy and update it
# using the filename they have provided
if fitsname is None:
fitsname = buildFITSName(_fname)
# Write out GEIS image as multi-extension FITS.
fexists = os.path.exists(fitsname)
if (fexists and clobber) or not fexists:
print('Writing out GEIS as MEF to ', fitsname)
fimg.writeto(fitsname, clobber=clobber)
if dqexists:
print('Writing out GEIS as MEF to ', dqfitsname)
dqfile.writeto(dqfitsname, clobber=clobber)
# Now close input GEIS image, and open writable
# handle to output FITS image instead...
fimg.close()
del fimg
# Image re-written as MEF, now it needs its WCS updated
updatewcs.updatewcs(fitsname)
fimg = fits.open(fitsname, mode=mode, memmap=memmap)
# Return handle for use by user
return fimg
def upwcs(f): print f updatewcs.updatewcs(f)
def runTweakReg(asn_file='GOODS-S-15-F140W_asn.fits', master_catalog='goodss_radec.dat', final_scale=0.06, ACS=False, threshold=5):
"""
Wrapper around tweakreg, generating source catalogs separately from
`findpars`.
"""
import glob
import shutil
import drizzlepac
from drizzlepac import tweakreg
from stwcs import updatewcs
import threedhst.prep_flt_astrodrizzle
asn = threedhst.utils.ASNFile(asn_file)
if ACS:
NCHIP=2
sci_ext = [1,4]
wht_ext = [2,5]
ext = 'flc'
dext = 'crclean'
else:
NCHIP=1
sci_ext = [1]
wht_ext = [2]
ext = 'flt'
dext = 'flt'
### Generate CRCLEAN images
for exp in asn.exposures:
updatewcs.updatewcs('%s_%s.fits' %(exp, ext))
has_crclean = True
for exp in asn.exposures:
has_crclean &= os.path.exists('%s_crclean.fits' %(exp))
threedhst.showMessage('# exposures: %d' %(len(asn.exposures)))
if not has_crclean:
if len(asn.exposures) == 1:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=False, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_cr_corr=False, driz_combine=True)
shutil.copy('%s_%s.fits' %(asn.exposures[0], ext), '%s_crclean.fits' %(asn.exposures[0]))
else:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=False, context=False, preserve=False, skysub=True, driz_separate=True, driz_sep_wcs=True, median=True, blot=True, driz_cr=True, driz_cr_corr=True, driz_combine=True)
#### Make SExtractor source catalogs in *each* flt
for exp in asn.exposures:
#updatewcs.updatewcs('%s_%s.fits' %(exp, ext))
for i in range(NCHIP):
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = '%s_%s.fits[%d]' %(exp, dext, wht_ext[i]-1)
se.options['WEIGHT_TYPE'] = 'MAP_RMS'
#
se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = '%s_%s_%d.cat' %(exp, ext, sci_ext[i])
se.options['FILTER'] = 'N'
se.options['DETECT_THRESH'] = '%f' %(threshold)
se.options['ANALYSIS_THRESH'] = '%f' %(threshold)
#
se.sextractImage('%s_%s.fits[%d]' %(exp, dext, sci_ext[i]-1))
threedhst.sex.sexcatRegions('%s_%s_%d.cat' %(exp, ext, sci_ext[i]), '%s_%s_%d.reg' %(exp, ext, sci_ext[i]), format=1)
#### TweakReg catfile
asn_root = asn_file.split('_asn')[0]
catfile = '%s.catfile' %(asn_root)
fp = open(catfile,'w')
for exp in asn.exposures:
line = '%s_%s.fits' %(exp, ext)
for i in range(NCHIP):
line += ' %s_%s_%d.cat' %(exp, ext, sci_ext[i])
fp.write(line + '\n')
fp.close()
#### First run AstroDrizzle mosaic
#drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True)
#### Make room for TWEAK wcsname
for exp in asn.exposures:
threedhst.prep_flt_astrodrizzle.clean_wcsname(flt='%s_%s.fits' %(exp, ext), wcsname='TWEAK', ACS=ACS)
#### Main run of TweakReg
if ACS:
refimage = '%s_drc_sci.fits' %(asn_root)
else:
refimage = '%s_drz_sci.fits' %(asn_root)
tweakreg.TweakReg(asn_file, refimage=refimage, updatehdr=True, updatewcs=True, catfile=catfile, xcol=2, ycol=3, xyunits='pixels', refcat=master_catalog, refxcol=1, refycol=2, refxyunits='degrees', shiftfile=True, outshifts='%s_shifts.txt' %(asn_root), outwcs='%s_wcs.fits' %(asn_root), searchrad=5, tolerance=12, wcsname='TWEAK', interactive=False, residplot='No plot', see2dplot=False, clean=True, headerlet=True, clobber=True)
#### Run AstroDrizzle again
if ACS:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, preserve=False)
else:
if len(asn.exposures) == 1:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale = '2.5 0.7', driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_cr_corr=False) # , final_wcs=True, final_rot=0)
else:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale = '2.5 0.7') # , final_wcs=True, final_rot=0)
for exp in asn.exposures:
files=glob.glob('%s*coo' %(exp))
files.extend(glob.glob('%s*crclean.fits' %(exp)))
for file in files:
os.remove(file)
def openImage(filename, mode='readonly', memmap=0, writefits=True,
clobber=True, fitsname=None):
"""
Opens file and returns PyFITS object. Works on both FITS and GEIS
formatted images.
Notes
-----
If a GEIS or waivered FITS image is used as input, it will convert it to a
MEF object and only if ``writefits = True`` will write it out to a file. If
``fitsname = None``, the name used to write out the new MEF file will be
created using `buildFITSName`.
Parameters
----------
filename: str
name of input file
mode: str
mode for opening file based on PyFITS `mode` parameter values
memmap: int
switch for using memory mapping, 0 for no, 1 for yes
writefits: bool
if True, will write out GEIS as multi-extension FITS
and return handle to that opened GEIS-derived MEF file
clobber: bool
overwrite previously written out GEIS-derived MEF file
fitsname: str
name to use for GEIS-derived MEF file,
if None and writefits==True, will use 'buildFITSName()' to generate one
"""
from stwcs import updatewcs
# Insure that the filename is always fully expanded
# This will not affect filenames without paths or
# filenames specified with extensions.
filename = osfn(filename)
# Extract the rootname and extension specification
# from input image name
_fname, _iextn = parseFilename(filename)
# Check whether we have a FITS file and if so what type
isfits, fitstype = isFits(_fname)
if isfits:
if fitstype != 'waiver':
# Open the FITS file
fimg = fits.open(_fname, mode=mode, memmap=memmap)
return fimg
else:
fimg = convertwaiveredfits.convertwaiveredfits(_fname)
#check for the existence of a data quality file
_dqname = buildNewRootname(_fname, extn='_c1f.fits')
dqexists = os.path.exists(_dqname)
if dqexists:
try:
dqfile = convertwaiveredfits.convertwaiveredfits(_dqname)
dqfitsname = buildNewRootname(_dqname, extn='_c1h.fits')
except:
print("Could not read data quality file %s" % _dqname)
if writefits:
# User wants to make a FITS copy and update it
# using the filename they have provided
if fitsname is None:
rname = buildNewRootname(_fname)
fitsname = buildNewRootname(rname, extn='_c0h.fits')
# Write out GEIS image as multi-extension FITS.
fexists = os.path.exists(fitsname)
if (fexists and clobber) or not fexists:
print('Writing out WAIVERED as MEF to ', fitsname)
fimg.writeto(fitsname, clobber=clobber)
if dqexists:
print('Writing out WAIVERED as MEF to ', dqfitsname)
dqfile.writeto(dqfitsname, clobber=clobber)
# Now close input GEIS image, and open writable
# handle to output FITS image instead...
fimg.close()
del fimg
# Image re-written as MEF, now it needs its WCS updated
updatewcs.updatewcs(fitsname)
fimg = fits.open(fitsname, mode=mode, memmap=memmap)
# Return handle for use by user
return fimg
else:
# Input was specified as a GEIS image, but no FITS copy
# exists. Read it in with 'readgeis' and make a copy
# then open the FITS copy...
try:
# Open as a GEIS image for reading only
fimg = readgeis.readgeis(_fname)
except:
raise IOError("Could not open GEIS input: %s" % _fname)
#check for the existence of a data quality file
_dqname = buildNewRootname(_fname, extn='.c1h')
dqexists = os.path.exists(_dqname)
if dqexists:
try:
dqfile = readgeis.readgeis(_dqname)
dqfitsname = buildFITSName(_dqname)
except:
print("Could not read data quality file %s" % _dqname)
# Check to see if user wanted to update GEIS header.
# or write out a multi-extension FITS file and return a handle to it
if writefits:
# User wants to make a FITS copy and update it
# using the filename they have provided
if fitsname is None:
fitsname = buildFITSName(_fname)
# Write out GEIS image as multi-extension FITS.
fexists = os.path.exists(fitsname)
if (fexists and clobber) or not fexists:
print('Writing out GEIS as MEF to ', fitsname)
fimg.writeto(fitsname, clobber=clobber)
if dqexists:
print('Writing out GEIS as MEF to ', dqfitsname)
dqfile.writeto(dqfitsname, clobber=clobber)
# Now close input GEIS image, and open writable
# handle to output FITS image instead...
fimg.close()
del fimg
# Image re-written as MEF, now it needs its WCS updated
updatewcs.updatewcs(fitsname)
fimg = fits.open(fitsname, mode=mode, memmap=memmap)
# Return handle for use by user
return fimg
def run(configobj):
""" Primary Python interface for image registration code
This task replaces 'tweakshifts'
"""
print('TweakReg Version %s(%s) started at: %s \n'%(
__version__,__version_date__,util._ptime()[0]))
util.print_pkg_versions()
# make sure 'updatewcs' is set to False when running from GUI or if missing
# from configObj:
if 'updatewcs' not in configobj:
configobj['updatewcs'] = False
# Check to see whether or not the imagefindpars parameters have
# already been loaded, as done through the python interface.
# Repeat for refimagefindpars
if PSET_SECTION not in configobj:
# Manage PSETs for source finding algorithms
_managePsets(configobj, PSET_SECTION, imagefindpars.__taskname__)
#print configobj[PSET_SECTION]
if PSET_SECTION_REFIMG not in configobj:
# Manage PSETs for source finding algorithms in reference image
_managePsets(configobj, PSET_SECTION_REFIMG,
refimagefindpars.__taskname__)
log.debug('')
log.debug("==== TweakReg was invoked with the following parameters: ====")
log.debug('')
util.print_cfg(configobj, log.debug)
# print out user set input parameter values for running this task
log.info('')
log.info("USER INPUT PARAMETERS common to all Processing Steps:")
util.printParams(configobj, log=log)
# start interpretation of input parameters
input_files = configobj['input']
# Start by interpreting the inputs
use_catfile = True
expand_refcat = configobj['expand_refcat']
enforce_user_order = configobj['enforce_user_order']
filenames, catnames = tweakutils.parse_input(
input_files, sort_wildcards=not enforce_user_order
)
catdict = {}
for indx,f in enumerate(filenames):
if catnames is not None and len(catnames) > 0:
catdict[f] = catnames[indx]
else:
catdict[f] = None
if not filenames:
print('No filenames matching input %r were found.' % input_files)
raise IOError
# Verify that files are writable (based on file permissions) so that
# they can be updated if either 'updatewcs' or 'updatehdr' have
# been turned on (2 cases which require updating the input files)
if configobj['updatewcs'] or configobj['UPDATE HEADER']['updatehdr']:
filenames = util.verifyFilePermissions(filenames)
if filenames is None or len(filenames) == 0:
raise IOError
if configobj['UPDATE HEADER']['updatehdr']:
wname = configobj['UPDATE HEADER']['wcsname']
# verify that a unique WCSNAME has been specified by the user
if not configobj['UPDATE HEADER']['reusename']:
for fname in filenames:
uniq = util.verifyUniqueWcsname(fname,wname)
if not uniq:
errstr = 'WCSNAME "%s" already present in "%s". '%(wname,fname)+\
'A unique value for the "wcsname" parameter needs to be ' + \
'specified. \n\nQuitting!'
print(textutil.textbox(errstr,width=60))
raise IOError
if configobj['updatewcs']:
print('\nRestoring WCS solutions to original state using updatewcs...\n')
updatewcs.updatewcs(filenames)
if catnames in [None,'',' ','INDEF'] or len(catnames) == 0:
catfile_par = configobj['COORDINATE FILE DESCRIPTION']['catfile']
# check to see whether the user specified input catalogs through other parameters
if catfile_par not in [None,'',' ','INDEF']:
# read in catalog file list provided by user
catnames,catdict = tweakutils.parse_atfile_cat('@'+catfile_par)
else:
use_catfile = False
if 'exclusions' in configobj and \
configobj['exclusions'] not in [None,'',' ','INDEF']:
if os.path.exists(configobj['exclusions']):
excl_files, excl_dict = tweakutils.parse_atfile_cat(
'@'+configobj['exclusions'])
# make sure the dictionary is well formed and that keys are base
# file names and that exclusion files have been expanded:
exclusion_files = []
exclusion_dict = {}
rootpath = os.path.abspath(
os.path.split(configobj['exclusions'])[0]
)
for f in excl_dict.keys():
print(f)
bf = os.path.basename(f)
exclusion_files.append(bf)
reglist = excl_dict[f]
if reglist is None:
exclusion_dict[bf] = None
continue
new_reglist = []
for regfile in reglist:
if regfile in [ None, 'None', '', ' ', 'INDEF' ]:
new_reglist.append(None)
else:
abs_regfile = os.path.normpath(
os.path.join(rootpath, regfile)
)
new_reglist.append(abs_regfile)
exclusion_dict[bf] = new_reglist
else:
raise IOError('Could not find specified exclusions file "{:s}"'
.format(configobj['exclusions']))
else:
exclusion_files = [None]*len(filenames)
exclusion_dict = {}
for f in filenames:
exclusion_dict[os.path.basename(f)] = None
# Verify that we have the same number of catalog files as input images
if catnames is not None and (len(catnames) > 0):
missed_files = []
for f in filenames:
if f not in catdict:
missed_files.append(f)
if len(missed_files) > 0:
print('The input catalogs does not contain entries for the following images:')
print(missed_files)
raise IOError
else:
# setup array of None values as input to catalog parameter for Image class
catnames = [None]*len(filenames)
use_catfile = False
# convert input images and any provided catalog file names into Image objects
input_images = []
# copy out only those parameters needed for Image class
catfile_kwargs = tweakutils.get_configobj_root(configobj)
# define default value for 'xyunits' assuming sources to be derived from image directly
catfile_kwargs['xyunits'] = 'pixels' # initialized here, required by Image class
del catfile_kwargs['exclusions']
if use_catfile:
# reset parameters based on parameter settings in this section
catfile_kwargs.update(configobj['COORDINATE FILE DESCRIPTION'])
for sort_par in imgclasses.sortKeys:
catfile_kwargs['sort_'+sort_par] = catfile_kwargs[sort_par]
# Update parameter set with 'SOURCE FINDING PARS' now
catfile_kwargs.update(configobj[PSET_SECTION])
uphdr_par = configobj['UPDATE HEADER']
hdrlet_par = configobj['HEADERLET CREATION']
objmatch_par = configobj['OBJECT MATCHING PARAMETERS']
catfit_pars = configobj['CATALOG FITTING PARAMETERS']
catfit_pars['minobj'] = objmatch_par['minobj']
objmatch_par['residplot'] = catfit_pars['residplot']
hdrlet_par.update(uphdr_par) # default hdrlet name
catfile_kwargs['updatehdr'] = uphdr_par['updatehdr']
shiftpars = configobj['OPTIONAL SHIFTFILE OUTPUT']
# verify a valid hdrname was provided, if headerlet was set to True
imgclasses.verify_hdrname(**hdrlet_par)
print('')
print('Finding shifts for: ')
for f in filenames:
print(' {}'.format(f))
print('')
log.info("USER INPUT PARAMETERS for finding sources for each input image:")
util.printParams(catfile_kwargs, log=log)
log.info('')
try:
minsources = max(1, catfit_pars['minobj'])
omitted_images = []
all_input_images = []
for imgnum in range(len(filenames)):
# Create Image instances for all input images
try:
regexcl = exclusion_dict[os.path.basename(filenames[imgnum])]
except KeyError:
regexcl = None
pass
img = imgclasses.Image(filenames[imgnum],
input_catalogs=catdict[filenames[imgnum]],
exclusions=regexcl,
**catfile_kwargs)
all_input_images.append(img)
if img.num_sources < minsources:
warn_str = "Image '{}' will not be aligned " \
"since it contains fewer than {} sources." \
.format(img.name, minsources)
print('\nWARNING: {}\n'.format(warn_str))
log.warning(warn_str)
omitted_images.append(img)
continue
input_images.append(img)
except KeyboardInterrupt:
for img in input_images:
img.close()
print('Quitting as a result of user request (Ctrl-C)...')
return
# create set of parameters to pass to RefImage class
kwargs = tweakutils.get_configobj_root(configobj)
# Determine a reference image or catalog and
# return the full list of RA/Dec positions
# Determine what WCS needs to be used for reference tangent plane
refcat_par = configobj['REFERENCE CATALOG DESCRIPTION']
if refcat_par['refcat'] not in [None,'',' ','INDEF']: # User specified a catalog to use
# Update kwargs with reference catalog parameters
kwargs.update(refcat_par)
# input_images list can be modified below.
# So, make a copy of the original:
input_images_orig_copy = copy(input_images)
do_match_refimg = False
# otherwise, extract the catalog from the first input image source list
if configobj['refimage'] not in [None, '',' ','INDEF']: # User specified an image to use
# A hack to allow different source finding parameters for
# the reference image:
ref_sourcefind_pars = \
tweakutils.get_configobj_root(configobj[PSET_SECTION_REFIMG])
ref_catfile_kwargs = catfile_kwargs.copy()
ref_catfile_kwargs.update(ref_sourcefind_pars)
ref_catfile_kwargs['updatehdr'] = False
log.info('')
log.info("USER INPUT PARAMETERS for finding sources for "
"the reference image:")
util.printParams(ref_catfile_kwargs, log=log)
#refimg = imgclasses.Image(configobj['refimage'],**catfile_kwargs)
# Check to see whether the user specified a separate catalog
# of reference source positions and replace default source list with it
if refcat_par['refcat'] not in [None,'',' ','INDEF']: # User specified a catalog to use
ref_source = refcat_par['refcat']
cat_src = ref_source
xycat = None
cat_src_type = 'catalog'
else:
try:
regexcl = exclusion_dict[configobj['refimage']]
except KeyError:
regexcl = None
pass
refimg = imgclasses.Image(configobj['refimage'],
exclusions=regexcl,
**ref_catfile_kwargs)
ref_source = refimg.all_radec
cat_src = None
xycat = refimg.xy_catalog
cat_src_type = 'image'
try:
if 'use_sharp_round' in ref_catfile_kwargs:
kwargs['use_sharp_round'] = ref_catfile_kwargs['use_sharp_round']
refimage = imgclasses.RefImage(configobj['refimage'],
ref_source, xycatalog=xycat,
cat_origin=cat_src, **kwargs)
refwcs_fname = refimage.wcs.filename
if cat_src is not None:
refimage.name = cat_src
except KeyboardInterrupt:
refimage.close()
for img in input_images:
img.close()
print('Quitting as a result of user request (Ctrl-C)...')
return
if len(input_images) < 1:
warn_str = "Fewer than two images are available for alignment. " \
"Quitting..."
print('\nWARNING: {}\n'.format(warn_str))
log.warning(warn_str)
for img in input_images:
img.close()
return
image = _max_overlap_image(refimage, input_images, expand_refcat,
enforce_user_order)
elif refcat_par['refcat'] not in [None,'',' ','INDEF']:
# a reference catalog is provided but not the reference image/wcs
if len(input_images) < 1:
warn_str = "No images available for alignment. Quitting..."
print('\nWARNING: {}\n'.format(warn_str))
log.warning(warn_str)
for img in input_images:
img.close()
return
if len(input_images) == 1:
image = input_images.pop(0)
else:
image, image2 = _max_overlap_pair(input_images, expand_refcat,
enforce_user_order)
input_images.insert(0, image2)
# Workaround the defect described in ticket:
# http://redink.stsci.edu/trac/ssb/stsci_python/ticket/1151
refwcs = []
for i in all_input_images:
refwcs.extend(i.get_wcs())
kwargs['ref_wcs_name'] = image.get_wcs()[0].filename
# A hack to allow different source finding parameters for
# the reference image:
ref_sourcefind_pars = \
tweakutils.get_configobj_root(configobj[PSET_SECTION_REFIMG])
ref_catfile_kwargs = catfile_kwargs.copy()
ref_catfile_kwargs.update(ref_sourcefind_pars)
ref_catfile_kwargs['updatehdr'] = False
log.info('')
log.info("USER INPUT PARAMETERS for finding sources for "
"the reference image (not used):")
util.printParams(ref_catfile_kwargs, log=log)
ref_source = refcat_par['refcat']
cat_src = ref_source
xycat = None
try:
if 'use_sharp_round' in ref_catfile_kwargs:
kwargs['use_sharp_round'] = ref_catfile_kwargs['use_sharp_round']
kwargs['find_bounding_polygon'] = True
refimage = imgclasses.RefImage(refwcs,
ref_source, xycatalog=xycat,
cat_origin=cat_src, **kwargs)
refwcs_fname = refimage.wcs.filename
refimage.name = cat_src
cat_src_type = 'catalog'
except KeyboardInterrupt:
refimage.close()
for img in input_images:
img.close()
print('Quitting as a result of user request (Ctrl-C)...')
return
else:
if len(input_images) < 2:
warn_str = "Fewer than two images available for alignment. " \
"Quitting..."
print('\nWARNING: {}\n'.format(warn_str))
log.warning(warn_str)
for img in input_images:
img.close()
return
kwargs['use_sharp_round'] = catfile_kwargs['use_sharp_round']
cat_src = None
refimg, image = _max_overlap_pair(input_images, expand_refcat,
enforce_user_order)
refwcs = []
#refwcs.extend(refimg.get_wcs())
#refwcs.extend(image.get_wcs())
#for i in input_images:
#refwcs.extend(i.get_wcs())
# Workaround the defect described in ticket:
# http://redink.stsci.edu/trac/ssb/stsci_python/ticket/1151
for i in all_input_images:
refwcs.extend(i.get_wcs())
kwargs['ref_wcs_name'] = refimg.get_wcs()[0].filename
try:
ref_source = refimg.all_radec
refimage = imgclasses.RefImage(refwcs, ref_source,
xycatalog=refimg.xy_catalog, **kwargs)
refwcs_fname = refimg.name
cat_src_type = 'image'
except KeyboardInterrupt:
refimage.close()
for img in input_images:
img.close()
print('Quitting as a result of user request (Ctrl-C)...')
return
omitted_images.insert(0, refimg) # refimage *must* be first
do_match_refimg = True
print("\n{0}\nPerforming alignment in the projection plane defined by the "
"WCS\nderived from '{1}'\n{0}\n".format('='*63, refwcs_fname))
if refimage.outxy is not None:
if cat_src is None:
cat_src = refimage.name
try:
log.info("USER INPUT PARAMETERS for matching sources:")
util.printParams(objmatch_par, log=log)
log.info('')
log.info("USER INPUT PARAMETERS for fitting source lists:")
util.printParams(configobj['CATALOG FITTING PARAMETERS'], log=log)
if hdrlet_par['headerlet']:
log.info('')
log.info("USER INPUT PARAMETERS for creating headerlets:")
util.printParams(hdrlet_par, log=log)
if shiftpars['shiftfile']:
log.info('')
log.info("USER INPUT PARAMETERS for creating a shiftfile:")
util.printParams(shiftpars, log=log)
# Now, apply reference WCS to each image's sky positions as well as the
# reference catalog sky positions,
# then perform the fit between the reference catalog positions and
# each image's positions
quit_immediately = False
xycat_lines = ''
xycat_filename = None
for img in input_images_orig_copy:
if xycat_filename is None:
xycat_filename = img.rootname+'_xy_catfile.list'
# Keep a record of all the generated input_xy catalogs
xycat_lines += img.get_xy_catnames()
retry_flags = len(input_images)*[0]
objmatch_par['cat_src_type'] = cat_src_type
while image is not None:
print ('\n'+'='*20)
print ('Performing fit for: {}\n'.format(image.name))
image.match(refimage, quiet_identity=False, **objmatch_par)
assert(len(retry_flags) == len(input_images))
if not image.goodmatch:
# we will try to match it again once reference catalog
# has expanded with new sources:
#if expand_refcat:
input_images.append(image)
retry_flags.append(1)
if len(retry_flags) > 0 and retry_flags[0] == 0:
retry_flags.pop(0)
image = input_images.pop(0)
# try to match next image in the list
continue
else:
# no new sources have been added to the reference
# catalog and we have already tried to match
# images to the existing reference catalog
#input_images.append(image) # <- add it back for later reporting
#retry_flags.append(1)
break
image.performFit(**catfit_pars)
if image.quit_immediately:
quit_immediately = True
image.close()
break
# add unmatched sources to the reference catalog
# (to expand it):
if expand_refcat:
refimage.append_not_matched_sources(image)
image.updateHeader(wcsname=uphdr_par['wcsname'],
reusename=uphdr_par['reusename'])
if hdrlet_par['headerlet']:
image.writeHeaderlet(**hdrlet_par)
if configobj['clean']:
image.clean()
image.close()
if refimage.dirty and len(input_images) > 0:
# The reference catalog has been updated with new sources.
# Clear retry flags and get next image:
image = _max_overlap_image(
refimage, input_images, expand_refcat,
enforce_user_order
)
retry_flags = len(input_images)*[0]
refimage.clear_dirty_flag()
elif len(input_images) > 0 and retry_flags[0] == 0:
retry_flags.pop(0)
image = input_images.pop(0)
else:
break
assert(len(retry_flags) == len(input_images))
if not quit_immediately:
# process images that have not been matched in order to
# update their headers:
si = 0
if do_match_refimg:
image = omitted_images[0]
image.match(refimage, quiet_identity=True, **objmatch_par)
si = 1
# process omitted (from start) images separately:
for image in omitted_images[si:]:
image.match(refimage, quiet_identity=False, **objmatch_par)
# add to the list of omitted images, images that could not
# be matched:
omitted_images.extend(input_images)
if len(input_images) > 0:
print("\nUnable to match the following images:")
print("-------------------------------------")
for image in input_images:
print(image.name)
print("")
# update headers:
for image in omitted_images:
image.performFit(**catfit_pars)
if image.quit_immediately:
quit_immediately = True
image.close()
break
image.updateHeader(wcsname=uphdr_par['wcsname'],
reusename=uphdr_par['reusename'])
if hdrlet_par['headerlet']:
image.writeHeaderlet(**hdrlet_par)
if configobj['clean']:
image.clean()
image.close()
if configobj['writecat'] and not configobj['clean']:
# Write out catalog file recording input XY catalogs used
# This file will be suitable for use as input to 'tweakreg'
# as the 'catfile' parameter
if os.path.exists(xycat_filename):
os.remove(xycat_filename)
f = open(xycat_filename, mode='w')
f.writelines(xycat_lines)
f.close()
if expand_refcat:
base_reg_name = os.path.splitext(
os.path.basename(cat_src))[0]
refimage.write_skycatalog(
'cumulative_sky_refcat_{:s}.coo' \
.format(base_reg_name),
show_flux=True, show_id=True
)
# write out shiftfile (if specified)
if shiftpars['shiftfile']:
tweakutils.write_shiftfile(input_images_orig_copy,
shiftpars['outshifts'],
outwcs=shiftpars['outwcs'])
except KeyboardInterrupt:
refimage.close()
for img in input_images_orig_copy:
img.close()
del img
print ('Quitting as a result of user request (Ctrl-C)...')
return
else:
print ('No valid sources in reference frame. Quitting...')
return
def subtract_flt_background(root='GOODN-N1-VBA-F105W', scattered_light=False, sex_background=False, order=2):
"""
Subtract polynomial background
"""
import scipy.optimize
import astropy.units as u
from astropy.table import Table as table
import stwcs
from stwcs import updatewcs
import drizzlepac
from drizzlepac import astrodrizzle, tweakreg, tweakback
import threedhst
asn = threedhst.utils.ASNFile(root+'_asn.fits')
for exp in asn.exposures:
updatewcs.updatewcs('%s_%s.fits' %(exp, 'flt'))
if not os.path.exists('%s_drz_sci.fits' %(root)):
if len(asn.exposures) == 1:
drizzlepac.astrodrizzle.AstroDrizzle(root+'_asn.fits', clean=False, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_cr_corr=False, driz_combine=True)
else:
drizzlepac.astrodrizzle.AstroDrizzle(root+'_asn.fits', clean=False, context=False, preserve=False, skysub=True, driz_separate=True, driz_sep_wcs=True, median=True, blot=True, driz_cr=True, driz_cr_corr=True, driz_combine=True)
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = '%s_drz_wht.fits' %(root)
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION,BACKGROUND'
se.options['CHECKIMAGE_NAME'] = '%s_drz_seg.fits,%s_drz_bkg.fits' %(root, root)
se.options['BACK_TYPE'] = 'AUTO'
se.options['BACK_SIZE'] = '256'
#
se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = '%s_drz_sci.cat' %(root)
se.options['FILTER'] = 'Y'
se.copyConvFile()
se.options['FILTER_NAME'] = 'gauss_4.0_7x7.conv'
se.options['DETECT_THRESH'] = '0.8'
se.options['ANALYSIS_THRESH'] = '0.8'
#
se.options['MEMORY_OBJSTACK'] = '30000'
se.options['MEMORY_PIXSTACK'] = '3000000'
se.options['MEMORY_BUFSIZE'] = '2048'
se.sextractImage('%s_drz_sci.fits' %(root))
#threedhst.sex.sexcatRegions('%s_flt.cat' %(exp), '%s_flt.reg' %(exp), format=1)
#### Blot segmentation map to FLT images for object mask
asn = threedhst.utils.ASNFile('%s_asn.fits' %(root))
#print 'Read files...'
ref = pyfits.open('%s_drz_sci.fits' %(root))
ref_wcs = stwcs.wcsutil.HSTWCS(ref, ext=0)
seg = pyfits.open('%s_drz_seg.fits' %(root))
#### Fill ref[0].data with zeros for seg mask
#seg_data = ref[0].data
#seg_data[seg[0].data == 0] = 0
seg_data = np.cast[np.float32](seg[0].data)
bkg_data = pyfits.open('%s_drz_bkg.fits' %(root))[0].data
yi, xi = np.indices((1014,1014))
if scattered_light:
bg_components = np.ones((4,1014,1014))
bg_components[1,:,:] = xi/1014.*2
bg_components[2,:,:] = yi/1014.*2
bg_components[3,:,:] = pyfits.open(os.getenv('THREEDHST') + '/CONF/G141_scattered_light.fits')[0].data
#### Use flat-field itself for images affected by full-field
#### persistence from the tungsten lamp
if scattered_light == 2:
bg_components[3,:,:] = pyfits.open(os.getenv('iref') + 'flat_UDF_F140W_v0.fits')[1].data[5:-5,5:-5]
NCOMP=4
else:
# bg_components = np.ones((3,1014,1014))
# bg_components[1,:,:] = xi/1014.*2
# bg_components[2,:,:] = yi/1014.*2
# NCOMP=3
#
if order == 2:
NCOMP=6
bg_components = np.ones((NCOMP,1014,1014))
bg_components[1,:,:] = (xi-507)/507.
bg_components[2,:,:] = (yi-507)/507.
bg_components[3,:,:] = ((xi-507)/507.)**2
bg_components[4,:,:] = ((yi-507)/507.)**2
bg_components[5,:,:] = (xi-507)*(yi-507)/507.**2
else:
NCOMP=3
bg_components = np.ones((NCOMP,1014,1014))
bg_components[1,:,:] = (xi-507)/507.
bg_components[2,:,:] = (yi-507)/507.
bg_flat = bg_components.reshape((NCOMP,1014**2))
#### Loop through FLTs, blotting reference and segmentation
models = []
for exp in asn.exposures:
flt = pyfits.open('%s_flt.fits' %(exp)) #, mode='update')
flt_wcs = stwcs.wcsutil.HSTWCS(flt, ext=1)
### segmentation
print 'Segmentation image: %s_blot.fits' %(exp)
blotted_seg = astrodrizzle.ablot.do_blot(seg_data, ref_wcs, flt_wcs, 1, coeffs=True, interp='nearest', sinscl=1.0, stepsize=10, wcsmap=None)
blotted_bkg = 0.
if sex_background:
blotted_bkg = astrodrizzle.ablot.do_blot(bkg_data, ref_wcs, flt_wcs, 1, coeffs=True, interp='nearest', sinscl=1.0, stepsize=10, wcsmap=None)
flt[1].data -= blotted_bkg
mask = (blotted_seg == 0) & (flt['DQ'].data == 0) & (flt[1].data > -1) & (xi > 10) & (yi > 10) & (xi < 1004) & (yi < 1004)
mask &= (flt[1].data < 5*np.median(flt[1].data[mask]))
data_range = np.percentile(flt[1].data[mask], [2.5, 97.5])
mask &= (flt[1].data >= data_range[0]) & (flt[1].data <= data_range[1])
data_range = np.percentile(flt[2].data[mask], [0.5, 99.5])
mask &= (flt[2].data >= data_range[0]) & (flt[2].data <= data_range[1])
### Least-sq fit for component normalizations
data = flt[1].data[mask].flatten()
wht = (1./flt[2].data[mask].flatten())**2
templates = bg_flat[:, mask.flatten()]
p0 = np.zeros(NCOMP)
p0[0] = np.median(data)
obj_fun = threedhst.grism_sky.obj_lstsq
popt = scipy.optimize.leastsq(obj_fun, p0, args=(data, templates, wht), full_output=True, ftol=1.49e-8/1000., xtol=1.49e-8/1000.)
xcoeff = popt[0]
model = np.dot(xcoeff, bg_flat).reshape((1014,1014))
models.append(model)
# add header keywords of the fit components
flt = pyfits.open('%s_flt.fits' %(exp), mode='update')
flt[1].data -= model+blotted_bkg
for i in range(NCOMP):
if 'BGCOMP%d' %(i+1) in flt[0].header:
flt[0].header['BGCOMP%d' %(i+1)] += xcoeff[i]
else:
flt[0].header['BGCOMP%d' %(i+1)] = xcoeff[i]
flt.flush()
coeff_str = ' '.join(['%.4f' %c for c in xcoeff])
threedhst.showMessage('Background subtraction, %s_flt.fits:\n\n %s' %(exp, coeff_str))
return models
def test_pixsky1(self):
"""This test verifies that the coordinate transformation tasks
'pixtopix', 'pixtosky', and 'skytopix' still work as expected.
This test relies on the truth/comparison file from the `acs_tweak` test
to define the output reference WCS for these coordinate transformations.
"""
input_file = self.get_input_file('input', 'j94f05bgq_flt.fits')
fpath, froot = os.path.split(input_file)
refwcs_file = self.get_data('truth', 'reference_tweak.fits')
ref_file1 = 'j94f05bgq_flt_reference.coo'
ref_file2 = 'j94f05bgq_flt_sky_reference.coo'
ref_file3 = 'j94f05bgq_flt_sci1_skyxy_reference.coo'
ref_file4 = 'j94f05bgq_flt_sci2_skyxy_reference.coo'
# Run test...
flt_file = updatewcs.updatewcs(froot)
# create combined results file
out_refxy_file = froot.replace('_flt.fits', '_flt_refxy.coo')
out_sky_file = froot.replace('_flt.fits', '_flt_sky.coo')
inskycat = froot.replace('_flt.fits', '_flt_sky_catalog.coo')
inskycat = self.get_data('input', inskycat)
outpixcat_names = []
outpix = open(out_refxy_file, 'w')
outsky = open(out_sky_file, 'w')
for i in [1, 2]:
outcat = froot.replace('_flt.fits', '_flt_sci%d_refxy.coo' % i)
incat = froot.replace('_flt.fits', '_flt_sci%d_xy_catalog.coo' % i)
incat = self.get_data('input', incat)
pixtopix.tran(froot+'[sci,%d]' % i, '{}[sci,1]'.format(refwcs_file),
direction='forward',
x=None, y=None, coordfile=incat, colnames='',
separator=None, precision=6, output=outcat,
verbose=False)
shutil.copyfileobj(open(outcat, 'r'), outpix)
outskycat = froot.replace('_flt.fits', '_flt_sci%d_sky.coo' % i)
pixtosky.xy2rd(froot+'[sci,%d]' % i, coordfile=incat,
x=None, y=None, colnames='', separator=None,
hms=True, precision=6, output=outskycat,
verbose=False)
shutil.copyfileobj(open(outskycat, 'r'), outsky)
outpixcat = froot.replace('_flt.fits', '_flt_sci%d_skyxy.coo' % i)
skytopix.rd2xy(froot+'[sci,%d]' % i, ra=None, dec=None,
coordfile=inskycat, colnames=None, precision=6,
output=outpixcat, verbose=False)
outpixcat_names.append(outpixcat)
# Close combined results files
outpix.close()
outsky.close()
outputs = [(out_refxy_file, ref_file1), (out_sky_file, ref_file2)]
for oname, rname in zip(outpixcat_names, [ref_file3, ref_file4]):
outputs.append((oname, rname))
self.compare_outputs(outputs)
def subtract_grism_background(asn_file='GDN1-G102_asn.fits', PATH_TO_RAW='../RAW/', final_scale=0.06, visit_sky=True, column_average=True, mask_grow=18, first_run=True, sky_iter=1):
"""
Subtract master grism sky from FLTs
"""
import os
import scipy.ndimage as nd
import pyregion
from drizzlepac import astrodrizzle
import drizzlepac
from stwcs import updatewcs
import stwcs
import threedhst.grism_sky as bg
asn = threedhst.utils.ASNFile(asn_file)
root = asn_file.split('_asn')[0]
sky_images = {'G141':['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light.fits'],
'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']}
#
# sky_images = {'G141':['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits', 'G141_scattered_light_v2.fits'],
# 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']}
# ### Don't use scattered light
# sky_images = {'G141':['zodi_G141_clean.fits', 'excess_lo_G141_clean.fits'],
# 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']}
#
# ## Use aXe images
# sky_images = {'G141':['WFC3.IR.G141.sky.V1.0.flat.fits', 'WFC3.IR.G141.sky.V1.0.flat.fits'],
# 'G102':['zodi_G102_clean.fits', 'excess_G102_clean.fits']}
if first_run:
### Rough background subtraction
threedhst.process_grism.fresh_flt_files(asn_file, from_path=PATH_TO_RAW, preserve_dq=False)
flt = pyfits.open('%s_flt.fits' %(asn.exposures[0]))
GRISM = flt[0].header['FILTER']
bg.set_grism_flat(grism=GRISM, verbose=True)
zodi = pyfits.open(os.getenv('THREEDHST')+'/CONF/%s' %(sky_images[GRISM][0]))[0].data
for exp in asn.exposures:
updatewcs.updatewcs('%s_flt.fits' %(exp))
flt = pyfits.open('%s_flt.fits' %(exp), mode='update')
#flt = pyfits.open('%s_flt.fits' %(exp))
flt[1].data *= bg.flat
#
mask = (flt['DQ'].data == 0)
data_range = np.percentile(flt[1].data[mask], [20, 80])
mask &= (flt[1].data >= data_range[0]) & (flt[1].data <= data_range[1]) & (flt[2].data != 0) & np.isfinite(flt[1].data) & np.isfinite(flt[2].data)
### Least-sq fit for component normalizations
data = flt[1].data[mask].flatten()
wht = (1./flt[2].data[mask].flatten())**2
zodi_mask = zodi[mask].flatten()
coeff_zodi = np.sum(data*zodi_mask*wht)/np.sum(zodi_mask**2*wht)
flt[1].data -= zodi*coeff_zodi
flt.flush()
threedhst.showMessage('Rough background for %s (zodi): %0.4f' %(exp, coeff_zodi))
#templates = bg_flat[:, mask.flatten()]
### Run astrodrizzle to make DRZ mosaic, grism-SExtractor mask
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, context=False, preserve=False, skysub=True, driz_separate=True, driz_sep_wcs=True, median=True, blot=True, driz_cr=True, driz_combine=True, final_wcs=False, resetbits=4096, final_bits=576, driz_sep_bits=576, driz_cr_snr='8.0 5.0', driz_cr_scale = '2.5 0.7')
else:
flt = pyfits.open('%s_flt.fits' %(asn.exposures[0]))
GRISM = flt[0].header['FILTER']
bg.set_grism_flat(grism=GRISM, verbose=True)
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = '%s_drz_wht.fits' %(root)
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION'
se.options['CHECKIMAGE_NAME'] = '%s_drz_seg.fits' %(root)
#
se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = '%s_drz_sci.cat' %(root)
se.options['FILTER'] = 'Y'
se.copyConvFile(grism=True)
se.options['FILTER_NAME'] = 'grism.conv'
se.options['DETECT_THRESH'] = '0.7'
se.options['ANALYSIS_THRESH'] = '0.7'
#
se.sextractImage('%s_drz_sci.fits' %(root))
#### Blot segmentation map to FLT images for object mask
ref = pyfits.open('%s_drz_sci.fits' %(root))
ref_wcs = stwcs.wcsutil.HSTWCS(ref, ext=0)
seg = pyfits.open('%s_drz_seg.fits' %(root))
seg_data = np.cast[np.float32](seg[0].data)
#### Loop through FLTs, blotting reference and segmentation
threedhst.showMessage('%s: Blotting grism segmentation masks.' %(root))
for exp in asn.exposures:
flt = pyfits.open('%s_flt.fits' %(exp))
flt_wcs = stwcs.wcsutil.HSTWCS(flt, ext=1)
### segmentation
#print 'Segmentation image: %s_blot.fits' %(exp)
blotted_seg = astrodrizzle.ablot.do_blot(seg_data, ref_wcs, flt_wcs, 1, coeffs=True, interp='nearest', sinscl=1.0, stepsize=10, wcsmap=None)
seg_grow = nd.maximum_filter((blotted_seg > 0)*1, size=8)
pyfits.writeto('%s_flt.seg.fits' %(exp), header=flt[1].header, data=seg_grow, clobber=True)
if first_run:
### Run background subtraction scripts
threedhst.process_grism.fresh_flt_files(asn_file, from_path=PATH_TO_RAW, preserve_dq=False)
for exp in asn.exposures:
updatewcs.updatewcs('%s_flt.fits' %(exp))
#threedhst.grism_sky.remove_grism_sky(flt=exp+'_flt.fits', list=sky_images[GRISM], path_to_sky=os.getenv('THREEDHST')+'/CONF/', verbose=True, second_pass=True, overall=True)
if visit_sky:
threedhst.grism_sky.remove_visit_sky(asn_file=asn_file, list=sky_images[GRISM], add_constant=False, column_average=(column_average) & (sky_iter == 1), mask_grow=mask_grow, flat_correct=first_run)
if (sky_iter > 1) & (~first_run):
for i in range(1, sky_iter):
threedhst.grism_sky.remove_visit_sky(asn_file=asn_file, list=sky_images[GRISM], add_constant=False, column_average=column_average & (i == (sky_iter-1)), mask_grow=mask_grow, flat_correct=False)
else:
for exp in asn.exposures:
threedhst.grism_sky.remove_grism_sky(flt='%s_flt.fits' %(exp), list=sky_images[GRISM], path_to_sky = os.getenv('THREEDHST')+'/CONF/', out_path='./', verbose=False, plot=False, flat_correct=first_run, sky_subtract=True, second_pass=column_average, overall=True, combine_skies=False, sky_components=True, add_constant=False)
### Astrodrizzle again to reflag CRs and make cleaned mosaic
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, skysub=False, skyuser='******', final_wcs=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale='2.5 0.7') # , final_wcs=True, final_rot=0)
def run(scifile,dgeofile=None,output=False,match_sci=False,update=True,vmin=None,vmax=None,plot_offset=0,plot_samp=32):
"""
This routine compares how well the sub-sampled DGEOFILE (generated
using the 'makesmall' module) corrects the input science image as
opposed to the full-size DGEOFILE.
SYNTAX:
import test_small_dgeo
test_small_dgeo.run(scifile,dgeofile=None,output=False)
where:
scifile - name of science image
dgeofile - name of full-sized DGEOFILE if not in DGEOFILE keyword
output - if True, write out differences to FITS file(s)
The user can either specify the full-size DGEOFILE reference filename
as the 'dgeofile' parameter or the code will look for the 'DGEOFILE'
keyword in the primary header for the name of the full-sized reference
file.
The primary output will be a series of plots showing the difference images
with the mean and stddev of the differences in the label of the image display.
If the 'output' parameter is set to True, these differences
will then be written out to FITS files based on the input science image
rootname. Both the DX and DY differences for a single chip will be written
out to the same file, with a separate file for each chip.
"""
from stsci.tools import fileutil
from stwcs import updatewcs
if update:
# update input SCI file to be consistent with reference files in header
print('Updating input file ',scifile,' to be consistent with reference files listed in header...')
updatewcs.updatewcs(scifile)
# Now, get the original NPOLFILE and overwrite the data in the scifile
# WCSDVARR extensions to remove the scaling by the linear terms imposed by
# the SIP convention
npolfile = fileutil.osfn(pyfits.getval(scifile,'NPOLFILE'))
npolroot = os.path.split(npolfile)[1]
dxextns = []
for extn in pyfits.open(npolfile):
if 'extname' in extn.header and extn.header['extname'] in ['DX','DY']:
dxextns.append([extn.header['extname'],extn.header['extver']])
#dxextns = [['dx',1],['dy',1],['dx',2],['dy',2]]
ndxextns = len(dxextns)
# Update input file with NPOLFILE arrays now
print('Updating input file ',scifile,' with original ')
print(' NPOLFILE arrays from ',npolfile)
fsci =pyfits.open(scifile,mode='update')
try:
next = fsci.index_of(('wcsdvarr',1))
except KeyError:
fsci.close()
print('=====')
print('ERROR: No WCSDVARR extensions found!')
print(' Please make sure NPOLFILE is specified and run this task with "update=True".')
print('=====')
return
# Replace WCSDVARR arrays here...
for dxe,wextn in zip(dxextns,list(range(1,ndxextns+1))):
fsci['wcsdvarr',wextn].data = pyfits.getdata(npolfile,dxe[0],dxe[1])
# Now replace the NPOLEXT keyword value with a new one so that it will automatically
# update with the correct file next time updatewcs is run.
fsci['sci',1].header['npolext'] = npolroot
print('Updated NPOLEXT with ',npolroot)
fsci.close()
print('\n=====')
print('WARNING: Updated file ',scifile,' NO LONGER conforms to SIP convention!')
print(' This file will need to be updated with updatewcs before using with MultiDrizzle.')
print('=====\n')
# Get info on full-size DGEOFILE
if dgeofile is None:
# read in full dgeofile from header
fulldgeofile = pyfits.getval(scifile,'DGEOFILE')
else:
fulldgeofile = dgeofile
print('Opening full-size DGEOFILE ',fulldgeofile,' for comparison.')
fulldgeofile = fileutil.osfn(fulldgeofile)
full_shape = [pyfits.getval(fulldgeofile,'NAXIS2','DX',1),pyfits.getval(fulldgeofile,'NAXIS1','DX',1)]
filter_names = fileutil.getFilterNames(pyfits.getheader(scifile))
detector = pyfits.getval(fulldgeofile,'DETECTOR')
# count the number of chips in DGEOFILE
xyfile = pyfits.open(scifile)
numchips = 0
ccdchip = []
extname = xyfile[1].header['EXTNAME']
for extn in xyfile:
if 'extname' in extn.header and extn.header['extname'] == extname:
numchips += 1
if 'ccdchip' in extn.header:
ccdchip.append(extn.header['ccdchip'])
else:
ccdchip.append(1)
if not match_sci:
ltv1 = 0
ltv2 = 0
nx = full_shape[1]
ny = full_shape[0]
else:
nx = xyfile['sci',1].header['NAXIS1']
ny = xyfile['sci',1].header['NAXIS2']
ltv1 = xyfile['sci',1].header['ltv1']
ltv2 = xyfile['sci',1].header['ltv2']
grid = [nx,ny,1]
print('grid of : ',nx,ny)
xyfile.close()
xarr,yarr = build_grid_arrays(nx,ny,1)
xgarr = xarr.reshape(grid[1],grid[0])
ygarr = yarr.reshape(grid[1],grid[0])
# initialize plot here
if has_matplotlib:
pl.clf()
pl.gray()
for chip,det in zip(list(range(1,numchips+1)),ccdchip):
xout,yout = transform_d2im_dgeo(scifile,chip,xarr,yarr)
dgeochip = 1
dgeo = pyfits.open(fulldgeofile)
for e in dgeo:
if 'ccdchip' not in e.header:
continue
else:
if e.header['ccdchip'] == det:
dgeochip = e.header['extver']
break
dgeo.close()
print('Matching sci,', chip, ' with DX,', dgeochip)
dx= (xout-xarr).reshape(grid[1],grid[0])
fulldatax = pyfits.getdata(fulldgeofile,'DX',dgeochip)
diffx=(dx-fulldatax[-ltv2:-ltv2+ny,-ltv1:-ltv1+nx]).astype(np.float32)
if has_matplotlib:
pl.imshow(diffx, vmin=vmin, vmax=vmax)
pl.title('dx-full_x: %s %s(DX,%d) with %g +/- %g' %
(filter_names, detector, dgeochip, diffx.mean(),
diffx.std()))
pl.colorbar()
if sys.version_info[0] < 3:
raw_input("Press 'ENTER' to close figure and plot DY...")
else:
input("Press 'ENTER' to close figure and plot DY...")
pl.close()
dy= (yout-yarr).reshape(grid[1],grid[0])
fulldatay = pyfits.getdata(fulldgeofile,'DY',dgeochip)
diffy=(dy-fulldatay[-ltv2:-ltv2+ny,-ltv1:-ltv1+nx]).astype(np.float32)
if has_matplotlib:
pl.imshow(diffy,vmin=vmin,vmax=vmax)
pl.title('dy-full_y: %s %s(DY,%d) with %g +/- %g ' %
(filter_names, detector, dgeochip, diffy.mean(),
diffy.std()))
pl.colorbar()
if sys.version_info[0] < 3:
raw_input("Press 'ENTER' to close figure and show next chip...")
else:
input("Press 'ENTER' to close figure and show next chip...")
pl.close()
if output:
# parse out rootname from input file if user wants results written to file
outroot = fileutil.buildNewRootname(scifile)
#
# setup DGEOFILE ref file as template for each chip's output results
# we only need dx,1 and dy,1 since each chip will be written out
# to a separate file and since we will use this template for
# writing out 2 different results files
#
fhdulist = pyfits.open(fulldgeofile)
hdulist = pyfits.HDUList()
hdulist.append(fhdulist[0])
hdulist.append(fhdulist['dx',1])
hdulist.append(fhdulist['dy',1])
fhdulist.close()
outname = outroot+'_sci'+str(chip)+'_dgeo_diffxy.match'
if os.path.exists(outname): os.remove(outname)
dxgarr = xgarr+diffx
dygarr = ygarr+diffy
wtraxyutils.write_xy_file(outname,[xgarr[plot_offset::plot_samp,plot_offset::plot_samp].flatten(),
ygarr[plot_offset::plot_samp,plot_offset::plot_samp].flatten(),
dxgarr[plot_offset::plot_samp,plot_offset::plot_samp].flatten(),
dygarr[plot_offset::plot_samp,plot_offset::plot_samp].flatten()],format="%20.8f",append=True)
outname = outroot+'_sci'+str(chip)+'_newfull_dxy.fits'
if os.path.exists(outname): os.remove(outname)
hdulist['dx',1].data = dx
hdulist['dy',1].data = dy
hdulist.writeto(outname)
outname = outroot+'_sci'+str(chip)+'_diff_dxy.fits'
if os.path.exists(outname): os.remove(outname)
hdulist['dx',1].data = diffx
hdulist['dy',1].data = diffy
hdulist.writeto(outname)
print('Created output file with differences named: ',outname)
del dx,dy,diffx,diffy
if output:
hdulist.close()
def prep_direct_grism_pair(direct_asn='goodss-34-F140W_asn.fits', grism_asn='goodss-34-G141_asn.fits', radec=None, raw_path='../RAW/', mask_grow=18, scattered_light=False, final_scale=None, skip_direct=False, ACS=False, jump=False, order=2, get_shift=True, align_threshold=20, column_average=True, sky_iter=3):
"""
Process both the direct and grism observations of a given visit
"""
import threedhst.prep_flt_astrodrizzle as prep
import drizzlepac
from stwcs import updatewcs
import time
t0 = time.time()
#direct_asn='goodss-34-F140W_asn.fits'; grism_asn='goodss-34-G141_asn.fits'; radec=None; raw_path='../RAW/'
#radec = os.getenv('THREEDHST') + '/ASTRODRIZZLE_FLT/Catalog/goodss_radec.dat'
################################
#### Direct image processing
################################
#### xx add astroquery 2MASS/SDSS workaround for radec=None
if not skip_direct:
#### Get fresh FLTS from ../RAW/
asn = threedhst.utils.ASNFile(direct_asn)
if ACS:
for exp in asn.exposures:
print 'cp %s/%s_flc.fits.gz .' %(raw_path, exp)
os.system('cp %s/%s_flc.fits.gz .' %(raw_path, exp))
os.system('gunzip %s_flc.fits.gz' %(exp))
else:
threedhst.process_grism.fresh_flt_files(direct_asn, from_path=raw_path)
if (not ACS):
#### Subtract WFC3/IR direct backgrounds
prep.subtract_flt_background(root=direct_asn.split('_asn')[0], scattered_light=scattered_light, order=order)
#### Flag IR CRs again within runTweakReg
#### Run TweakReg
if (radec is None) & (not ACS):
drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=0.8, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7') # ,
else:
if get_shift:
prep.runTweakReg(asn_file=direct_asn, master_catalog=radec, final_scale=None, ACS=ACS, threshold=align_threshold)
#### Subtract background of direct ACS images
if ACS:
for exp in asn.exposures:
flc = pyfits.open('%s_flc.fits' %(exp), mode='update')
for ext in [1,4]:
threedhst.showMessage('Subtract background from %s_flc.fits[%d] : %.4f' %(exp, ext, flc[ext].header['MDRIZSKY']))
flc[ext].data -= flc[ext].header['MDRIZSKY']
flc[ext].header['MDRIZSK0'] = flc[ext].header['MDRIZSKY']
flc[ext].header['MDRIZSKY'] = 0.
#
flc.flush()
else:
pass
#### Do this later, gives segfaults here???
#prep.subtract_flt_background(root=direct_asn.split('_asn')[0], scattered_light=scattered_light)
#### Flag CRs again on BG-subtracted image
#drizzlepac.astrodrizzle.AstroDrizzle(direct_asn, clean=True, final_scale=None, final_pixfrac=0.8, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7') # ,
################################
#### Grism image processing
################################
if grism_asn:
asn = threedhst.utils.ASNFile(grism_asn)
if ACS:
for exp in asn.exposures:
print 'cp %s/%s_flc.fits.gz .' %(raw_path, exp)
os.system('cp %s/%s_flc.fits.gz .' %(raw_path, exp))
os.system('gunzip %s_flc.fits.gz' %(exp))
updatewcs.updatewcs('%s_flc.fits' %(exp))
prep.copy_adriz_headerlets(direct_asn=direct_asn, grism_asn=grism_asn, ACS=True)
prep.subtract_acs_grism_background(asn_file=grism_asn, final_scale=None)
else:
#### Remove the sky and flag CRs
## with mask from rough zodi-only subtraction
prep.subtract_grism_background(asn_file=grism_asn, PATH_TO_RAW='../RAW/', final_scale=None, visit_sky=True, column_average=False, mask_grow=mask_grow, first_run=True)
## Redo making mask from better combined image
prep.subtract_grism_background(asn_file=grism_asn, PATH_TO_RAW='../RAW/', final_scale=final_scale, visit_sky=True, column_average=column_average, mask_grow=mask_grow, first_run=False, sky_iter=sky_iter)
#### Copy headers from direct images
if radec is not None:
prep.copy_adriz_headerlets(direct_asn=direct_asn, grism_asn=grism_asn, ACS=False)
#### Run CR rejection with final shifts
drizzlepac.astrodrizzle.AstroDrizzle(grism_asn, clean=True, skysub=False, final_wcs=True, final_scale=final_scale, final_pixfrac=0.8, context=False, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale='2.5 0.7') # driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7')
if not grism_asn:
t1 = time.time()
threedhst.showMessage('direct: %s\n\nDone (%d s).' %(direct_asn, int(t1-t0)))
else:
t1 = time.time()
threedhst.showMessage('direct: %s\ngrism: %s\n\nDone (%d s).' %(direct_asn, grism_asn, int(t1-t0)))
def auto_run(root='j023507-040202', flag_global_crs=False):
import os
import glob
import numpy as np
import astropy.io.fits as pyfits
import astropy.wcs as pywcs
from drizzlepac import updatehdr
from stwcs import updatewcs
from grizli import utils, prep
from grizli.pipeline import auto_script
utils.set_warnings()
visit_file = '{0}_visits.npy'.format(root)
visits, all_groups, info = np.load(visit_file)
# Something wrong with some files with bad shifts, reset wcs
for visit in visits:
for file in visit['files']:
utils.fetch_hst_calibs(
file, calib_types=['IDCTAB', 'NPOLFILE', 'IMPHTTAB'])
updatewcs.updatewcs(file, verbose=True, use_db=False)
# Apply shifts
shift_log = '{0}_shifts.log'.format(visit['product'])
if os.path.exists(shift_log):
sh = utils.read_catalog(shift_log)
flt0 = pyfits.open(sh['flt'][0])
wcs_ref = pywcs.WCS(flt0['SCI', 1].header, fobj=flt0, relax=True)
shift_dict = {}
for i in range(len(sh)):
shift_dict[sh['flt'][i]] = [sh['xshift'][i], sh['yshift'][i]]
prep.apply_tweak_shifts(wcs_ref,
shift_dict,
grism_matches={},
verbose=False)
# Redrizzle mosaics
prep.drizzle_overlaps(visits,
check_overlaps=False,
skysub=False,
static=False,
pixfrac=0.5,
scale=None,
final_wcs=False,
fetch_flats=False,
final_rot=None,
include_saturated=True)
####### Alignment
os.system('rm *wcs.*')
# Radec
master_radec = '{0}/../../{1}_master.radec'.format(os.getcwd(), root)
if not os.path.exists(master_radec):
master_radec = None
ref_catalog = 'USER'
if root.startswith('cos-'):
hsc = '{0}/../../{1}'.format(os.getcwd(),
'hsc-udeep-i25_corr_cosmos.radec')
if os.path.exists(hsc):
master_radec = hsc
ref_catalog = 'HSC'
elif root.startswith('uds-'):
hsc = '{0}/../../{1}'.format(os.getcwd(),
'hsc-udeep-sxds_corr_uds.radec')
if os.path.exists(hsc):
master_radec = hsc
ref_catalog = 'HSC'
parent_radec = '{0}/../../{1}_parent.radec'.format(os.getcwd(), root)
if not os.path.exists(parent_radec):
parent_radec = None
if master_radec is not None:
radec = master_radec
elif parent_radec is not None:
radec = parent_radec
else:
radec = None
if radec is None:
needs_gaia = True
else:
needs_gaia = False
REFERENCE = 'GAIA'
REFERENCE = 'PS1'
print('master RADEC file: ', radec)
thresh = 2.5
for visit in visits:
# Clean catalogs
files = glob.glob('{0}.*'.format(visit['product']))
for file in files:
os.remove(file)
# Generate GAIA alignment catalog at the observation epoch
clip = 120
clip = -1
if needs_gaia:
flt = pyfits.open(visit['files'][0])
h = flt['SCI', 1].header
ra_i, dec_i = h['CRVAL1'], h['CRVAL2']
radec, ref_catalog = prep.get_radec_catalog(
ra=ra_i,
dec=dec_i,
product=visit['product'],
date=flt[0].header['EXPSTART'],
date_format='mjd',
reference_catalogs=[REFERENCE],
radius=5.)
flt.close()
if REFERENCE == 'GAIA':
mag_limits = [16, 20]
else:
mag_limits = [18, 22]
#clip = 50
if '_flc' in visit['files'][0]:
triangle_size_limit = [5, 4000 * np.sqrt(2)]
else:
triangle_size_limit = [5, 1300]
else:
mag_limits = [19, 23]
triangle_size_limit = [5, 1300]
# Remake catalogs
cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh)
# Redo alignment
try:
print('XXX clip', clip, mag_limits, triangle_size_limit)
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=mag_limits,
simple=False,
max_err_percentile=80,
clip=clip,
outlier_threshold=5,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
except:
print('First align failed! Relax parameters')
try:
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=[10, 20],
simple=False,
max_err_percentile=99,
clip=160,
outlier_threshold=20,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
except:
try:
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=[10, 20],
simple=False,
max_err_percentile=99,
clip=160,
outlier_threshold=40,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
except:
radec = '{0}_ps1.radec'.format(visit['product'])
ref_catalog = 'PS1'
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=mag_limits,
simple=False,
max_err_percentile=80,
clip=120,
outlier_threshold=5,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
#continue
orig_wcs, drz_wcs, out_shift, out_rot, out_scale = result
# Propagate shifts
for file in visit['files']:
updatehdr.updatewcs_with_shift(file,
str('{0}_wcs.fits'.format(
visit['product'])),
xsh=out_shift[0],
ysh=out_shift[1],
rot=out_rot,
scale=out_scale,
wcsname=ref_catalog,
force=True,
reusename=True,
verbose=True,
sciext='SCI')
### Bug in astrodrizzle? Dies if the FLT files don't have MJD-OBS
### keywords
im = pyfits.open(file, mode='update')
im[0].header['MJD-OBS'] = im[0].header['EXPSTART']
im.flush()
# Redrizzle mosaics again including new shifts
prep.drizzle_overlaps(visits,
check_overlaps=False,
skysub=False,
static=False,
pixfrac=0.8,
scale=None,
final_wcs=False,
fetch_flats=False,
final_rot=None)
# Remake catalogs
thresh = 2.5
for visit in visits:
# Remake catalogs
cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh)
prep.table_to_regions(cat, '{0}.cat.reg'.format(visit['product']))
prep.table_to_radec(cat, '{0}.cat.radec'.format(visit['product']))
# Update visits file
v = auto_script.get_visit_exposure_footprints(visit_file=visit_file,
check_paths=['./', '../RAW'],
simplify=1.e-6)
if flag_global_crs:
# Assume everything at same orient
pass
if False:
# Mosaic
auto_script.drizzle_overlaps(root,
filters=['F160W'],
min_nexp=1,
pixfrac=0.8,
scale=0.1,
make_combined=False,
ref_image=None,
static=False)
def imaging_pipeline(root_filename, output_path = None, cr_reject_switch=True,
astrodrizzle_switch=True, png_switch=True, reproc_switch=False):
'''
This is the main controller for all the steps in the pipeline.
'''
# Get information from the header
header_data = get_metadata(root_filename)
detector = header_data['detector']
dateobs = header_data['dateobs']
# Generate the output filenames
filename = os.path.abspath(root_filename)
output_file_dict = make_output_file_dict(root_filename,header_data)
# Run CR reject
if cr_reject_switch:
output_check = check_for_outputs(output_file_dict['cr_reject_output'][1])
if reproc_switch == False and output_check == True:
print 'Not reprocessing cr_reject files.'
logging.info("Not reprocessing cr_reject files.")
else:
logging.info("Running cr_reject")
print 'Running cr_reject'
cosmicx_params = get_cosmicx_params(header_data)
logging.info(cosmicx_params)
output_filename = output_file_dict['cr_reject_output'][1]
run_cosmicx(root_filename,
output_filename,
cosmicx_params,
detector)
print 'Done running cr_reject'
logging.info("Done running cr_reject")
else:
logging.info("Skipping cr_reject ")
print 'Skipping cr_reject'
# Run astrodrizzle.
if astrodrizzle_switch:
output_check = check_for_outputs(output_file_dict['drizzle_output'])
if reproc_switch == False and output_check == True:
logging.info("Not reprocessing astrodrizzle files.")
print 'Not reprocessing astrodrizzle files.'
else:
logging.info("Running Astrodrizzle")
print 'Running Astrodrizzle'
for filename, output in zip(output_file_dict['cr_reject_output'],
output_file_dict['drizzle_output']):
updatewcs.updatewcs(filename)
run_astrodrizzle(filename, output, detector, dateobs)
print 'Done running astrodrizzle'
logging.info("Done running astrodrizzle")
else:
print 'Skipping astrodrizzle'
logging.info("Skipping astrodrizzle")
# Run trim.
if png_switch:
output_check = check_for_outputs(output_file_dict['png_output'])
if reproc_switch == False and output_check == True:
print 'Not reprocessing png files.'
logging.info("Not reprocessing png files.")
else:
print 'Running png'
logging.info("Running png")
for filename in output_file_dict['drizzle_output']:
run_trim(filename, output_path,log_switch=True)
print 'Done running png'
logging.info("Done running png")
else:
print 'Skipping running png'
logging.info("Skipping running png")
