def align_images(input_catalogs,
reference_catalog=None,
searchrad=None,
gaia=False,
**kwargs):
"""
Run TweakReg on the images, using the catalogs from hst1pass.
Astrometrically align the input images via TweakReg. Uses the
catalogs from hstpass as the input positions of the sources. Can
optionally align images to a reference frame by giving a reference
catalog such as Gaia. For more information, see the TweakReg
documentation.
Parameters
----------
input_catalogs : list
List of catalog filenames from hst1pass (strings).
reference_catalog : str, optional
File name of the reference catalog. If None images are aligned
to first image in input_images.
searchrad : float, optional
Search radius in arcseconds, if not given, uses tweakreg
default of 1 arcsec.
gaia : bool
Align images to Gaia? If True, queries Gaia in region
encompassing input images, and saves catalog of Gaia source
positions, and uses this catalog in TweakReg
**kwargs : keyword arguments, optional
Other keyword arguments to be passed to TweakReg. Examples
could include minflux, maxflux, fluxunits. More information
available at
https://drizzlepac.readthedocs.io/en/latest/tweakreg.html
"""
from drizzlepac import tweakreg
input_images = [os.path.splitext(cat)[0] + '.fits' \
for cat in input_catalogs]
make_tweakreg_catfile(input_images)
if gaia:
if reference_catalog:
print('Gaia set to true, overriding reference catalog \
with Gaia catalog')
reference_catalog = get_gaia_cat(input_images)
tweakreg.TweakReg(input_images,
catfile='tweakreg_catlist.txt',
refcat=reference_catalog,
searchrad=searchrad,
interactive=False,
updatehdr=True,
shiftfile=True,
reusename=True,
clean=True,
**kwargs)
print('Updating Catalogs with new RA/Dec')
make_chip_catalogs(input_catalogs)
def align(expnames, **kwargs):
""" Align exposures to a GAIA-based source catalog
Parameters
===========
expnames : str or list of strings
Filename of exposure or list of filenames to be aligned to GAIA catalog
Returns
=======
gaia_catalog : Table
Astropy Table object containing gaia catalog retrieved for exposures
shift_name : string
Filename of shift file written out by `tweakreg.Tweakreg`
"""
setname = os.path.basename(expnames[0]).split('_')[0]
shift_name = kwargs.get('shift_name',None)
if shift_name is None:
shift_name = '{}_shifts_gaia.txt'.format(setname)
ref_cat_file = kwargs.get('ref_cat_file', '{}_gaia_ref.cat'.format(setname))
# Set default values for specific Tweakreg parameters which are more
# appropriate for most of our use cases
updatehdr = kwargs.get('updatehdr', False)
wcsname = kwargs.get('wcsname', "TWEAK_GAIA")
searchrad = kwargs.get('searchrad', 15.0)
searchunits = kwargs.get('searchunits', 'arcseconds')
threshold = kwargs.get('threshold', 1000.0)
conv_width = kwargs.get('conv_width', 3.5)
gaia_catalog = hlautils.create_astrometric_catalog(expnames, **kwargs)
gaia_catalog.write(ref_cat_file, format='ascii.no_header', overwrite=True)
print("FINISHED writing out gaia_catalog: {}".format(os.path.abspath(ref_cat_file)))
if len(gaia_catalog) > 6:
fitgeometry = 'general'
else:
fitgeometry = 'shift'
tweakreg.TweakReg(expnames,
shiftfile=True,
interactive=False,
clean=True,
see2dplot=False,
updatehdr=updatehdr,
wcsname=wcsname,
threshold=threshold,
conv_width=conv_width,
outshifts=shift_name,
outwcs = shift_name.replace('.txt','_wcs.fits'),
refcat=ref_cat_file,
searchrad=searchrad,
searchunits=searchunits,
fitgeometry=fitgeometry)
return gaia_catalog, shift_name
def myalign(input_files,
updatewcs=False, clean=True, logfile=None, verbose=False,
wcsname='ALIGN_ALL', refimage=None,
**kwargs):
logger = RegularPrint() if verbose else PrintRedirect(logfile)
if isinstance(input_files, str):
input_files = [input_files]
already_updated = []
for file in input_files:
if 'WCSMAN' not in fits.getheader(file, extname='PRIMARY'):
fits.setval(file, 'WCSMAN', 'PRIMARY', value=False,
comment='Has WCS been manually updated?')
# Only update WCS if has not been performed on ANY of the inputs
elif fits.getval(file, 'WCSMAN', 'PRIMARY') is True:
already_updated.append(file.rpartition('/')[-1])
if updatewcs:
if len(already_updated) > 0:
updatewcs = False
print('--NOT UPDATING WCS because manually updated files exist: '
f'{already_updated}')
else:
print('--Updating WCS manually as requested')
# Set default options
t_kwgs = {
'conv_width': 4.0,
'threshold': 200,
'shiftfile': False,
'updatehdr': True,
'writecat': False,
'clean': True,
'residplot': 'NoPlot',
'see2dplot': False,
'reusename': True,
'interactive': False,
'updatewcs': updatewcs,
'enforce_user_order': True,
'expand_refcat': True,
'wcsname': wcsname,
'refimage': (refimage or ''),
}
t_kwgs.update(kwargs)
# Initial Alignment
with logger:
print(t_kwgs)
tweakreg.TweakReg(input_files, **t_kwgs)
if t_kwgs['updatewcs']:
for file in input_files:
fits.setval(file, 'WCSMAN', 'PRIMARY', value=True)
def mytweakreg_quad_main(path, refim):
os.chdir(path)
os.system('pwd')
updatewcs.updatewcs('*flc.fits')
teal.unlearn('tweakreg')
findpars = {
'computesig': True,
'threshold': 100.,
'conv_width': 3.5,
'peakmin': 100.,
'peakmax': 90000.,
'nsigma': 1.5,
'ratio': 1.0,
'theta': 0.0,
'dqbits': None,
'use_sharp_round': False,
'ylimit': 0.1
}
tweakreg.TweakReg(glob.glob('*flc.fits'),
expand_refcat=False,
enforce_user_order=False,
updatehdr=False,
shiftfile=True,
writecat=True,
clean=False,
interactive=True,
verbose=False,
headerlet=False,
minobj=15,
searchrad=250.0,
searchunits='pixels',
use2dhist=True,
see2dplot=True,
separation=0.5,
fitgeometry='rscale',
residplot='both',
labelsize=8,
nclip=5,
sigma=3.0,
refimage=refim,
imagefindcfg=findpars,
refimagefindcfg=findpars)
def calc_shift(im, radec):
hdr = fits.getheader(im)
det = hdr['DETECTOR']
if det == 'WFC':
cw = 3.5
elif det == 'UVIS':
cw = 3.5 * (0.04 / 0.05)
elif det == 'IR':
cw = 2.5 * (0.13 / .1)
tweakreg.TweakReg(im,
updatehdr=False,
expand_refcat=False,
enforce_user_order=True,
imagefindcfg={
'threshold': 5.,
'conv_width': cw
},
refcat=radec,
shiftfile=True,
outshifts='single_shift.txt',
outwcs='hsc_shifts_wcs.fits',
refimage=None)
def align(expnames, **kwargs):
""" Align exposures to a GAIA-based source catalog
Parameters
===========
expnames : str or list of strings
Filename of exposure or list of filenames to be aligned to GAIA catalog
"""
shift_name = kwargs.get('shift_name', 'shifts_gaia.txt')
updatehdr = kwargs.get('updatehdr', False)
wcsname = kwargs.get('wcsname', "TWEAK_GAIA")
searchrad = kwargs.get('searchrad', 15.0)
searchunits = kwargs.get('searchunits', 'arcseconds')
ref_cat_file = kwargs.get('output', None)
gaia_catalog = astrometric_utils.create_astrometric_catalog(
expnames, **kwargs)
gaia_catalog.write(ref_cat_file, format='ascii.no_header')
if len(gaia_catalog) > 6:
fitgeometry = 'general'
else:
fitgeometry = 'shift'
tweakreg.TweakReg(expnames,
shiftfile=True,
interactive=False,
clean=True,
see2dplot=False,
updatehdr=updatehdr,
wcsname=wcsname,
outshifts=shift_name,
outwcs=shift_name.replace('.txt', '_wcs.fits'),
refcat=ref_cat_file,
searchrad=searchrad,
searchunits=searchunits,
fitgeometry=fitgeometry)
return gaia_catalog
def run_tweakreg_alignment(self, reffile, tweakfile, workdir):
cwd=os.getcwd()
# MdB: do the tweakreg in a temp directory
os.chdir(workdir)
try:
os.mkdir('shifts')
except Exception as e:
print e
os.chdir('shifts')
# MdB: not sure what should have been put in here
imagefind_dict = {}
fitgeometry=self.fitgeometry
# MdB: now we finally run tweakreg
if self.verbose:
print "Running tweakreg on ",tweakfile
print "and reference ",reffile
#tweakreg.TweakReg(tweakfile,minobj=10,xoffset=0,yoffset=0, imagefindcfg=imagefind_dict, refimage=reffile, searchrad=10., shiftfile=True, outshifts='shift.txt', see2dplot=False, residplot='No plot')
tweakreg.TweakReg(tweakfile, refimage=reffile, searchrad=2., shiftfile=True, outshifts='shift.txt', see2dplot=False, residplot='No plot', updatehdr=False, fitgeometry=fitgeometry,minobj=4,tolerance=3.0)#,tolerance=100,rfluxunits='cps')
os.system('cp shift.txt ..')
# MdB: move back to the original directory
os.chdir(cwd)
def run_tweakreg(images, options):
if fits_utils.get_instrument(images[0]) != 'wfc3_ir_full':
# Run cosmic ray rejection on the input images to make registering the images easier
for image in images:
# Make a copy of the original data file to use later
shutil.copy(image, make_raw_tmp_filename(image))
# Remove cosmic rays, this operates on the files in place
detect_cosmic_rays(image, options['crpars'])
tweakreg.TweakReg(files=images,
refimage=options['refimage'],
interactive=False,
writecat=False,
clean=True,
updatehdr=True,
wcsname='TWEAK',
reusename=True,
rfluxunits='counts',
see2dplot=False,
separation=0.5,
residplot="No plot",
runfile='',
imagefindcfg={
'threshold': options['tweakshifts_threshold'],
'use_sharp_round': True
},
refimagefindcfg={
'threshold': options['tweakshifts_threshold'],
'use_sharp_round': True
})
if fits_utils.get_instrument(images[0]) != 'wfc3_ir_full':
for image in images:
# copy the raw data back into the input file with the updated shift
copy_data(make_raw_tmp_filename(image), image)
# Clean up
os.remove(make_raw_tmp_filename(image))
def align_drizzled(images=['MACS2129-35-F814W_drc_sci.fits', 'MACS2129-36-F814W_drc_sci.fits']):
from astropy.table import Table as table
from drizzlepac import astrodrizzle, tweakreg, tweakback
for image in images:
root = image.split('_sci.fits')[0]
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = '%s_wht.fits[0]' %(root)
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
#
se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = '%s_sci.cat' %(root)
se.options['FILTER'] = 'N'
se.options['DETECT_THRESH'] = '5'
se.options['ANALYSIS_THRESH'] = '5'
#
se.sextractImage('%s_sci.fits[0]' %(root))
threedhst.sex.sexcatRegions('%s_sci.cat' %(root), '%s_sci.reg' %(root), format=1)
#
t = table.read('%s_sci.cat' %(root), format='ascii.sextractor')
np.savetxt('%s_sci.xy' %(root), np.array([t['X_IMAGE'], t['Y_IMAGE']]).T, fmt='%.7f')
fp = open('%s_sci.catfile' %(root), 'w')
fp.write('%s_sci.fits %s_sci.xy\n' %(root, root))
fp.close()
reference = '%s_sci.xy' %(images[0].split('_sci.fits')[0])
for image in images[1:]:
root = image.split('_sci.fits')[0]
tweakreg.TweakReg(image, refimage=images[0], updatehdr=True, updatewcs=True, catfile='%s_sci.catfile' %(root), xcol=1, ycol=2, xyunits='pixels', refcat=reference, refxcol=1, refycol=2, refxyunits='pixels', shiftfile=False, searchrad=5, tolerance=12, wcsname='TWEAK3', interactive=False, residplot='No plot', see2dplot=False, clean=True, headerlet=True, clobber=True)
tweakback.tweakback(image)
pass
def test_tweak(self):
""" This test confirms that image alignment performed by tweakreg
works to find the correct alignment solution and that the solution
can be used to correctly update the headers of the inputs to generate
a properly aligned, combined mosaic using astrodrizzle.
"""
input_file1 = 'j94f05bgq_flt.fits'
input_file2 = 'j9irw1rqq_flt.fits'
output = 'acs_tweak'
# Prepare input files.
input_file1 = self.get_input_file('input', input_file1)
input_file2 = self.get_input_file('input', input_file2)
inputs = ','.join([input_file1, input_file2])
tweak_parObj = teal.load('tweakreg', defaults=True)
tweak_parObj['input'] = 'j94f05bgq_flt.fits,j9irw1rqq_flt.fits'
tweak_parObj['writecat'] = False
tweak_parObj['clean'] = True
tweak_parObj['OBJECT MATCHING PARAMETERS']['searchrad'] = 3.0
tweak_parObj['OBJECT MATCHING PARAMETERS']['see2dplot'] = False
tweak_parObj['OBJECT MATCHING PARAMETERS']['separation'] = 0.0
tweak_parObj['CATALOG FITTING PARAMETERS']['residplot'] = 'No plot'
tweak_parObj['CATALOG FITTING PARAMETERS']['nclip'] = 5
tweak_parObj['CATALOG FITTING PARAMETERS']['sigma'] = 2.8
imfind_parObj = teal.load('imagefindpars', defaults=True)
imfind_parObj['computesig'] = False
imfind_parObj['skysigma'] = 20.0
imfind_parObj['conv_width'] = 2.0
imfind_parObj['threshold'] = 500.0
imfind_parObj['dqbits'] = None
rfind_parObj = teal.load('refimagefindpars', defaults=True)
rfind_parObj['computesig'] = False
rfind_parObj['skysigma'] = 20.0
rfind_parObj['conv_width'] = 2.0
rfind_parObj['threshold'] = 500.0
rfind_parObj['dqbits'] = None
tweakreg.TweakReg(files=inputs, configobj=tweak_parObj,
imagefindcfg=imfind_parObj,
refimagefindcfg=rfind_parObj,
updatehdr=True, wcsname='TWEAK_regress')
# run astrodrizzle now...
adriz_parObj = teal.load('astrodrizzle', defaults=True) # get all default values
adriz_parObj['output'] = 'acs_tweak'
adriz_parObj['build'] = True
adriz_parObj['in_memory'] = True
adriz_parObj['runfile'] = 'drizzlepac.run'
adriz_parObj['STATE OF INPUT FILES']['preserve'] = False
adriz_parObj['STATE OF INPUT FILES']['clean'] = True
adriz_parObj['STEP 2: SKY SUBTRACTION']['skywidth'] = 0.3
adriz_parObj['STEP 2: SKY SUBTRACTION']['use_static'] = False
adriz_parObj['STEP 2: SKY SUBTRACTION']['sky_bits'] = None
adriz_parObj['STEP 4: CREATE MEDIAN IMAGE']['combine_maskpt'] = 0.7
astrodrizzle.AstroDrizzle(inputs, configobj=adriz_parObj)
# Compare results
outfile = '{}_drz.fits'.format(output)
reffile = 'reference_tweak.fits'
outputs = [(outfile, reffile)]
self.compare_outputs(outputs)
def do_tweak(flt_list, besti, lowthreshold=0):
f = open(bad_pix_list_wfc3)
lines = f.read().split('\n')
f.close()
lines = [item.split(None) for item in lines]
lines = [item for item in lines if item != []]
bad_pix = [(int(item[0]), int(item[1])) for item in lines]
tmp_ims = []
for i in range(len(flt_list)):
f = pyfits.open(flt_list[i])
if f[0].header["INSTRUME"] == "ACS":
tmp_ims.append(flt_list[i].replace(".fits", "_lac.fits"))
acs = True
else:
tmp_ims.append(flt_list[i].replace(".fits", "_filter.fits"))
if flt_list[i] == tmp_ims[i]:
print "Error with ", flt_list[i]
sys.exit(1)
print "Median Filtering ", flt_list[i]
f = pyfits.open(flt_list[i])
tmpdata = copy.deepcopy(f["SCI"].data)
LTV1 = f["SCI"].header["LTV1"]
LTV2 = f["SCI"].header["LTV2"]
for this_x, this_y in bad_pix:
this_x += LTV1
this_y += LTV2
if this_x > 1 and this_x < len(
tmpdata[0]) and this_y > 1 and this_y < len(tmpdata):
f["SCI"].data[
int(np.around(this_y - 1)),
int(np.around(this_x - 1))] = np.median(
tmpdata[int(np.around(this_y -
2)):int(np.around(this_y +
1)),
int(np.around(this_x -
2)):int(np.around(this_x +
1))])
f.writeto(tmp_ims[i], clobber=True)
f.close()
acs = False
do_it("cp -f " + tmp_ims[i] + " " +
tmp_ims[i].replace("/orig_files/", "/"))
tmp_ims[i] = tmp_ims[i].replace("/orig_files/", "/")
print "tmp_ims ", tmp_ims
tweakref = tmp_ims[besti]
tweakreg.TweakReg(
','.join(tmp_ims),
updatehdr=True,
shiftfile=True, # This is just for show
############ Change This Between Iterations: ##########
refimage=tweakref,
updatewcs=False, # I think this should always be false.
searchrad=2,
searchunits='arcseconds',
threshold=(1. + 7. * acs) / (lowthreshold + 1.),
conv_width=(2.5 + 1. * acs), # 3.5 for optical, 2.5 for IR
######### Change This Between Iterations: ##############
wcsname="TWEAK_rough",
residplot='No plot',
see2dplot=False,
fitgeometry='shift'
) # Have to change this for that one epoch, G cluster?
f = open("shifts.txt")
lines = f.read()
f.close()
if lines.find(" nan ") != -1:
print "Couldn't match!"
if lowthreshold == 0: # First iteration
print "Trying lower threshold..."
do_tweak(flt_list, besti, lowthreshold=1)
else:
print "...even though lowthreshold is ", lowthreshold
sys.exit(1)
for i in range(len(flt_list)):
print "Transferring from ", tmp_ims[i], flt_list[i]
transfer_header(tmp_ims[i], flt_list[i])
def runTweakReg(asn_file='GOODS-S-15-F140W_asn.fits', master_catalog='goodss_radec.dat', final_scale=0.06, ACS=False, threshold=5):
"""
Wrapper around tweakreg, generating source catalogs separately from
`findpars`.
"""
import glob
import shutil
import drizzlepac
from drizzlepac import tweakreg
from stwcs import updatewcs
import threedhst.prep_flt_astrodrizzle
asn = threedhst.utils.ASNFile(asn_file)
if ACS:
NCHIP=2
sci_ext = [1,4]
wht_ext = [2,5]
ext = 'flc'
dext = 'crclean'
else:
NCHIP=1
sci_ext = [1]
wht_ext = [2]
ext = 'flt'
dext = 'flt'
### Generate CRCLEAN images
for exp in asn.exposures:
updatewcs.updatewcs('%s_%s.fits' %(exp, ext))
has_crclean = True
for exp in asn.exposures:
has_crclean &= os.path.exists('%s_crclean.fits' %(exp))
threedhst.showMessage('# exposures: %d' %(len(asn.exposures)))
if not has_crclean:
if len(asn.exposures) == 1:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=False, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_cr_corr=False, driz_combine=True)
shutil.copy('%s_%s.fits' %(asn.exposures[0], ext), '%s_crclean.fits' %(asn.exposures[0]))
else:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=False, context=False, preserve=False, skysub=True, driz_separate=True, driz_sep_wcs=True, median=True, blot=True, driz_cr=True, driz_cr_corr=True, driz_combine=True)
#### Make SExtractor source catalogs in *each* flt
for exp in asn.exposures:
#updatewcs.updatewcs('%s_%s.fits' %(exp, ext))
for i in range(NCHIP):
se = threedhst.sex.SExtractor()
se.options['WEIGHT_IMAGE'] = '%s_%s.fits[%d]' %(exp, dext, wht_ext[i]-1)
se.options['WEIGHT_TYPE'] = 'MAP_RMS'
#
se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True
se.params['MAG_AUTO'] = True
#
se.options['CATALOG_NAME'] = '%s_%s_%d.cat' %(exp, ext, sci_ext[i])
se.options['FILTER'] = 'N'
se.options['DETECT_THRESH'] = '%f' %(threshold)
se.options['ANALYSIS_THRESH'] = '%f' %(threshold)
#
se.sextractImage('%s_%s.fits[%d]' %(exp, dext, sci_ext[i]-1))
threedhst.sex.sexcatRegions('%s_%s_%d.cat' %(exp, ext, sci_ext[i]), '%s_%s_%d.reg' %(exp, ext, sci_ext[i]), format=1)
#### TweakReg catfile
asn_root = asn_file.split('_asn')[0]
catfile = '%s.catfile' %(asn_root)
fp = open(catfile,'w')
for exp in asn.exposures:
line = '%s_%s.fits' %(exp, ext)
for i in range(NCHIP):
line += ' %s_%s_%d.cat' %(exp, ext, sci_ext[i])
fp.write(line + '\n')
fp.close()
#### First run AstroDrizzle mosaic
#drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True)
#### Make room for TWEAK wcsname
for exp in asn.exposures:
threedhst.prep_flt_astrodrizzle.clean_wcsname(flt='%s_%s.fits' %(exp, ext), wcsname='TWEAK', ACS=ACS)
#### Main run of TweakReg
if ACS:
refimage = '%s_drc_sci.fits' %(asn_root)
else:
refimage = '%s_drz_sci.fits' %(asn_root)
tweakreg.TweakReg(asn_file, refimage=refimage, updatehdr=True, updatewcs=True, catfile=catfile, xcol=2, ycol=3, xyunits='pixels', refcat=master_catalog, refxcol=1, refycol=2, refxyunits='degrees', shiftfile=True, outshifts='%s_shifts.txt' %(asn_root), outwcs='%s_wcs.fits' %(asn_root), searchrad=5, tolerance=12, wcsname='TWEAK', interactive=False, residplot='No plot', see2dplot=False, clean=True, headerlet=True, clobber=True)
#### Run AstroDrizzle again
if ACS:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, preserve=False)
else:
if len(asn.exposures) == 1:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale = '2.5 0.7', driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_cr_corr=False) # , final_wcs=True, final_rot=0)
else:
drizzlepac.astrodrizzle.AstroDrizzle(asn_file, clean=True, final_scale=final_scale, final_pixfrac=0.8, context=False, resetbits=4096, final_bits=576, driz_sep_bits=576, preserve=False, driz_cr_snr='8.0 5.0', driz_cr_scale = '2.5 0.7') # , final_wcs=True, final_rot=0)
for exp in asn.exposures:
files=glob.glob('%s*coo' %(exp))
files.extend(glob.glob('%s*crclean.fits' %(exp)))
for file in files:
os.remove(file)
def align(self,
subtract_background=True,
align_method=None,
ref_catalog=None,
create_diff_source_lists=True,
updatehdr=True,
updatewcs=True,
wcsname='DASH',
threshold=50.,
cw=3.5,
searchrad=20.,
astrodriz=True,
cat_file='catalogs/diff_catfile.cat',
drz_output=None,
move_files=False):
'''
Aligns new FLT's to reference catalog.
Parameters
----------
self : object
DashData object created from an individual IMA file.
subtract_background : bool, optional
If True, runs subtract_background_reads functions.
align_method : string, optional
Defines alignment method to be used. Default is None (input files
will align to each other).
ref_catalog : cat file, optional
Defines reference image that will be referenced for CATALOG
alignment method.
create_diff_source_lists : bool, optional
Specifies whether or not to create a segmentation image and source
list from the difference files
updatehdr : bool, optional
Specifies whether to update the headers after aligning during
TweakReg. Default is True.
updatewcs : bool, optional
Specifies whether to update the WCS after aligning during TweakReg.
Default is True.
wcsname : str, optional (as long as name you choose doesn't already exist)
Specifies name of WCS. Default is 'DASH'
threshold : float, optional
cw : float, optional
searchrad : float, optional
Radius (in pixels) that TweakReg will search around sources to find
matches. Default is 20 pixels.
astrodriz : bool, optional
Specifies whether to drizzle images together using Astrodrizzle.
Default is True.
cat_file : str, optional
Name of catfile to be used to align sources in TweakReg. Default is
the catfile created by setting create_diff_source_lists to True,
catalogs/diff_catfile.cat
drz_output : str, optional
Name of output file after drizzling using AstroDrizzle. Default is
the root name of the original IMA.
Outputs
-------
Shifts file : txt file
File containing shifts during TweakReg
WCS Shifts file : fits file
File containing WCS shifts during TweakReg
Residual plot : png
Plot showing residuals from TweakReg alignment
Vector plot : png
Plot showing vector poitnings from TweakReg alignment
Aligned FLT's : fits file
Same diff files created in split_ima that have been aligned using TweakReg
Drizzled Image : fits file
Setting astrodriz to True will output a drizzled image form a single
exposure (produced only if astrodriz is set to True)
'''
#Set name for output drizzled image to the rootname of the original IMA if it is not specified
if drz_output is None:
drz_output = self.root
input_images = sorted(glob('diff/{}_*_diff.fits'.format(self.root)))
if not os.path.exists('shifts'):
os.mkdir('shifts')
outshifts = 'shifts/shifts_{}.txt'.format(self.root)
outwcs = 'shifts/shifts_{}_wcs.fits'.format(self.root)
if subtract_background:
self.subtract_background_reads()
#Align images to a catalog
if align_method == 'CATALOG':
if (ref_catalog is not None):
##Create source list and segmentation maps based on difference files
if create_diff_source_lists is True:
diffpath = os.path.dirname(
os.path.abspath('diff/{}_*_diff.fits'.format(
self.root)))
cat_images = sorted([
os.path.basename(x)
for x in glob('diff/{}_*_diff.fits'.format(self.root))
])
sc_diff_files = [diffpath + '/' + s for s in cat_images]
self.diff_seg_map(cat_images=sc_diff_files)
teal.unlearn('tweakreg')
teal.unlearn('imagefindpars')
tweakreg.TweakReg(input_images,
refcat=ref_catalog,
catfile=cat_file,
xcol=2,
ycol=3,
updatehdr=updatehdr,
updatewcs=updatewcs,
wcsname=wcsname,
verbose=True,
imagefindcfg={
'threshold': threshold,
'conv_width': cw
},
searchrad=searchrad,
searchunits='pixels',
shiftfile=True,
outshifts=outshifts,
outwcs=outwcs,
interactive=False,
fitgeometry='rscale',
minobj=5)
else:
raise Exception(
'Need to specify reference catalog and reference image.')
#Align images to the first image
else:
teal.unlearn('tweakreg')
teal.unlearn('imagefindpars')
tweakreg.TweakReg(input_images,
catfile=cat_file,
xcol=2,
ycol=3,
updatehdr=updatehdr,
updatewcs=updatewcs,
wcsname=wcsname,
verbose=True,
imagefindcfg={
'threshold': threshold,
'conv_width': cw
},
searchrad=searchrad,
searchunits='pixels',
shiftfile=True,
outshifts=outshifts,
outwcs=outwcs,
interactive=False,
fitgeometry='rscale',
minobj=5)
pass
#Drizzle the images together
if astrodriz is True:
#Do not have drizzle take 256 flags into account
no_tfs = 2, 4, 8, 16, 32, 64, 128, 512, 2048, 4096, 8192, 16384
astrodrizzle.AstroDrizzle(input_images,
output=drz_output,
clean=False,
final_pixfrac=1.0,
context=False,
resetbits=0,
preserve=False,
driz_cr_snr='8.0 5.0',
driz_cr_scale='2.5 0.7',
driz_sep_bits=no_tfs,
final_bits=no_tfs)
if move_files is True:
self.move_files()
def drizzle_fields( obsDates, fltList, cluster, filter, search_rad=1, thresh=1):
'''
PURPOSE : IF THERE ARE OBSERVATIONS FROM DIFFERENT DATES / OBSRUNS
THEN DRIZZLE THE DATES THAT ARE THE SAME TOGETHER
THEN TWEAK THEM TO ONE REFERENCE DRIZZLE IMAGE
TWEAK BACK TO THE FLT FITS
INPUTS : OBSDATES : CAN BE ANY UNIQUE IDENTIFIER TO NAME OF THE RUN
OR DATE
thresh=1.0
search_rad=1.0 #Often this parameter is too big if the images hardly overlap. Keep at 1 if limited sources
'''
outputfilename = cluster+'_'+filter
uniqueObs = np.unique(np.array(obsDates))
print 'MORE THAN ONE OBSERVATION DATE/RUN SO DRIZZLING INDIVIDUALLY'
print uniqueObs
call(["mkdir", "keep"])
for iDate in uniqueObs:
obsDrizzle = np.array(fltList)[ np.array(obsDates) == str(iDate) ]
drizzleString = ",".join(obsDrizzle)
print iDate
print obsDrizzle
if not os.path.isfile("keep/"+str(iDate)+"_drz_sci.fits"):
astrodrizzle.AstroDrizzle(drizzleString, \
output=str(iDate), \
final_wcs=True, \
final_scale=0.03, \
final_pixfrac=0.8, \
combine_type='iminmed')
call(["cp",str(iDate)+"_drz_sci.fits","keep"])
else:
call(["cp","keep/"+str(iDate)+"_drz_sci.fits","."])
#3. NOW TWEAK EACH DRIZZLED IMAGE
#-----------------------------------------
#First sextract them using tweak_sextract
drzList=[]
for iDate in uniqueObs:
drzList.append(str(iDate)+"_drz_sci.fits")
try:
drzList.remove(str(outputfilename)+'_drz_sci.fits')
except:
print 'its okay, cluster drizzle doesn exist'
#Run sextractor on the images
tweaksex.tweakreg_sextract(drzList,'catfile')
refDate=obsDates[1]#
refimage=drzList[1]#
drzString= ",".join(drzList)
tweaksex.ref_sex( refimage )
tweakreg.TweakReg(drzString, \
residplot='both', \
see2dplot=True, \
threshold=np.float(thresh), \
searchrad=np.float(search_rad), \
fitgeometry='rscale', updatehdr='True', \
refimage=refimage,
catfile='catfile', \
xcol=1, ycol=2, fluxcol=3, \
minobj=2, \
refcat='reference.cat', \
refxcol=1, refycol=2, rfluxcol=3)
for iDate in uniqueObs:
if iDate != obsDates:
TweakFLT= np.array(fltList)[ np.array(obsDates) == str(iDate) ]
tweakString = ",".join(TweakFLT)
print tweakString
tweakback.tweakback( str(iDate)+'_drz_sci.fits', \
input=tweakString )
def run_tweakreg(imagename, updatehdr=True, sort_files=False):
""" Runs tweakreg.TweakReg and updates WCS in headers.
Actually first runs on the FLCs and applied the same WCS solution on
the corresponding FLTs. This is especially important in later data
because CTI gets to be so bad that tweakreg may not find enough sources
in FLTs with short exposure time in the narrow-band filter.
Parameters:
imagename : string
Name of the image.
updatehdr : {True, False}
Set on to update WCS in header. Usually should have this on
unless testing stuff!!
sort_files : {True, False}
Turn on if want to sort the output files into proposal
directories.
Returns:
nothing.
Outputs:
In the 'data/newdata' directory:
* A WCS-corrected FLC.
* A WCS-corrected FLT.
Moved to the 'outputs/tweakreg' directory:
* A matched tweakreg catalog ipppssoot_xy_catalog.match
* Another matched tweakreg catalog ipppssoot_flc_catalog_fit.match
(we actually use the first one, but retain this one just in case)
* A WCS solution file ipppssoot_hlet.fits, which is used to tweak
the FLT.
* The tweakreg log ipppssoot_flc_tweakreg.log.
* The diagnoistic plots hist2d_ipppssoot_flc.png
residuals_ipppssoot_flc.png
vector_ipppssoot_flc.png
All other tweakreg outputs are removed.
Notes from R. Avila:
By setting headerlet=True, attach=False, creates a file called ipppssoot_hlet.fits
that contains the new solution. you then run a task called apply_headerlet
which can apply the headerlet to the FLT
"""
if updatehdr:
headerlet = True
else:
headerlet = False
# Obtain the filter, target, expotime, and proposal number.
priheader = fits.getheader(imagename, 0)
targname = (priheader['TARGNAME']).lower()
proposid = str(priheader['PROPOSID'])
filt = (priheader['FILTER']).lower()
exptime = int(priheader['EXPTIME'])
chinject = str(priheader['CHINJECT'])
# Set the path to the proposal directory.
path_to_data = config.path_to_data
proposaldir = os.path.join(path_to_data, proposid)
# Get rootname.
rootname = imagename.split('_fl')[0]
imagenames = [rootname+'_flt.fits', rootname+'_flc.fits']
# Skip the charge injections of CONT from 12348 (tweakreg can't find any sources)
if chinject == 'CONT':
for imagename in imagenames:
shutil.move(imagename, proposaldir)
print("Charge injection CONT!")
print("Moved {} to {}".format(imagename, proposaldir))
return ''
# Get path to the master catalogs.
path_to_data = config.path_to_data
path_to_mastercat = os.path.join(path_to_data, 'master_cat')
# Change parameters based on target, filter, and exptime.
if '6791' in targname:
print('ngc6791')
master_image = os.path.join(path_to_mastercat, 'out_n6791_master_drc_sci.fits')
master_cat = os.path.join(os.path.join(path_to_mastercat, 'final'), 'ngc6791_master.cat')
if filt == 'f606w':
print('f606w')
if exptime <= 60: #short exp
print(exptime)
conv_width = 3.5
threshold = 25
peakmax = 50000
elif exptime > 60: #long exp
conv_width = 3.5
threshold = 200
peakmax = 50000
elif filt == 'f502n':
print('f502n')
if exptime <= 60: #short exp
print(exptime)
conv_width = 3.5
threshold = 5
peakmax = 50000
elif exptime > 60: #long exp
print(exptime)
conv_width = 3.5
threshold = 100
peakmax = 50000
elif '104' in targname:
print('ngc104')
master_image = os.path.join(path_to_mastercat, 'out_n104_master_drc_sci.fits')
master_cat = os.path.join(os.path.join(path_to_mastercat, 'final'), 'ngc104_master.cat')
if filt == 'f606w':
print('f606w')
if exptime <= 60: #short exp
print(exptime)
conv_width = 3.5
threshold = 25
peakmax = 50000
elif exptime > 60: #long exp
print(exptime)
conv_width = 3.5
threshold = 200
peakmax = 50000
elif filt == 'f502n':
print('f502n')
if exptime <= 60: #short exp
print(exptime)
conv_width = 3.5
threshold = 5
peakmax = 50000
elif exptime > 60: #long exp
print(exptime)
conv_width = 3.5
threshold = 100
peakmax = 50000
elif '6583' in targname:
print('ngc6583')
master_image = os.path.join(path_to_mastercat, 'out_n6583_master_drc_sci.fits')
master_cat = os.path.join(os.path.join(path_to_mastercat, 'final'), 'ngc6583_master.cat')
conv_width = 3.5
threshold = 25
peakmax = 50000
tweakreg.TweakReg(imagename,
updatehdr=updatehdr,
headerlet=headerlet,
attach=False,
wcsname='TWEAK',
reusename=True,
interactive=False,
conv_width=conv_width,
threshold=threshold,
minobj=5,
peakmax=peakmax,
refcat=master_cat,
refimage=master_image,
refxcol=2,
refycol=3,
refxyunits='pixels',
runfile=imagename+'_tweakreg.log')
# Apply the WCS change to the FLT using the *hlet file.
if updatehdr:
hlet.apply_headerlet_as_primary(imagenames[0],
'{}_hlet.fits'.format(rootname),
attach=False,
archive=False)
# how get rid of this warning?
#WARNING: AstropyDeprecationWarning: Deletion of non-existent keyword 'DISTNAME': In a future Astropy version Header.__delitem__ may be changed so that this raises a KeyError just like a dict would. Please update your code so that KeyErrors are caught and handled when deleting non-existent keywords. [astropy.io.fits.header]
# Move the files to the proposal directory.
if sort_files:
for imagename in imagenames:
shutil.move(imagename, proposaldir)
print("Moved {} to {}".format(imagename, proposaldir))
path_to_outputs = config.path_to_outputs
path_to_tweakreg = os.path.join(path_to_outputs, 'tweakreg')
# Move the *.match to outputs/tweakreg.
match_files = glob.glob('*'+rootname+'*.match')
for match_file in match_files:
if os.path.isfile(os.path.join(path_to_tweakreg, match_file)):
os.remove(os.path.join(path_to_tweakreg, match_file))
shutil.move(match_file, path_to_tweakreg)
print("Moved {} to {}".format(match_file, path_to_tweakreg))
# Move the hlet file to outputs/tweakreg
if updatehdr:
if os.path.isfile(os.path.join(path_to_tweakreg,
'{}_hlet.fits'.format(rootname))):
os.remove(os.path.join(path_to_tweakreg,
'{}_hlet.fits'.format(rootname)))
shutil.move('{}_hlet.fits'.format(rootname), path_to_tweakreg)
print("Moved {} to {}".format('{}_hlet.fits'.format(rootname), path_to_tweakreg))
# Move the tweakreg png files to outputs/tweakreg
png_files = glob.glob('*'+rootname+'*.png')
for png_file in png_files:
if os.path.isfile(os.path.join(path_to_tweakreg, png_file)):
os.remove(os.path.join(path_to_tweakreg, png_file))
shutil.move(png_file, path_to_tweakreg)
print("Moved {} to {}".format(png_file, path_to_tweakreg))
# Move the log files to outputs/tweakreg
if os.path.isfile(os.path.join(path_to_tweakreg,
'{}_flc.fits_tweakreg.log'.format(rootname))):
os.remove(os.path.join(path_to_tweakreg,
'{}_flc.fits_tweakreg.log'.format(rootname)))
shutil.move('{}_flc.fits_tweakreg.log'.format(rootname), path_to_tweakreg)
print("Moved {} to {}".format('{}_flc.fits_tweakreg.log'.format(rootname),
path_to_tweakreg))
f.write(ref_lines[0])
g = open('catalog_'+non_ref[i]+'.list')
nref_lines = g.readlines()
g.close()
for j in np.arange(len(nref_lines)):
f.write(nref_lines[j])
f.close()
print("... TWEAKREG task for "+ip.ref_flt+"(ref), "+non_ref[i]+" ...")
# ----- Running tweakreg task ----- #
twk_order = [ip.ref_flt, comb_name, non_ref[i]]
for twk in twk_order:
tweakreg.TweakReg('@input_'+twk+'.list', interactive=False, updatehdr=True,
runfile='tweakreg_'+twk+'.log', wcsname='TWEAK_'+twk+'_'+str(i+1),
catfile='catalog_'+twk+'.list',
xcol=1, ycol=2, fluxcol=3, minobj=ip.n_minobj, searchrad=ip.tolerance,
tolerance=2.0)
# ----- After tweakreg task ----- #
os.system('rm -rfv *catalog.coo *.match *xy*.list')
# Copying header-updated images to drizzle path
for inp in lst_input:
flt = inp.split('input')[1].split('.')[0]
if not (flt == '_'+comb_name):
print('# ----- filter '+flt[1:]+' ----- #')
flc = np.genfromtxt(inp, dtype=None, encoding='ascii')
for file in flc:
os.system('cp -rpv '+file+' drz'+flt+'/')
if not os.path.exists('./originals/'):
os.mkdir('./originals/')
shutil.copy('narrowband.fits', './originals/narrowband.fits')
shutil.copy('broadband.fits', './originals/broadband.fits')
narrowband_hdu = fits.open('narrowband.fits')[0]
broadband_hdu = fits.open('broadband.fits')[0]
broadband_median = np.nanmedian(broadband_hdu.data)
narrowband_median = np.nanmedian(narrowband_hdu.data)
tweakreg.TweakReg('broadband.fits',
imagefindcfg={
'threshold': 3 * broadband_median,
'conv_width': 3.5
},
refimage='narrowband.fits',
refimagefindcfg={
'threshold': 3 * narrowband_median,
'conv_width': 3.5
},
updatehdr=True,
shiftfile=True,
outshifts='shift.txt')
narrowband_hdu = fits.open('narrowband.fits')[0]
broadband_hdu = fits.open('broadband.fits')[0]
reprojected_data, _ = reproject_interp(broadband_hdu, narrowband_hdu.header)
broadband_hdu_updated_wcs = narrowband_hdu.copy()
broadband_hdu_updated_wcs.data = reprojected_data
broadband_hdu_updated_wcs.writeto('broadband_regrid.fits', overwrite=True)
os.system(
def tweak_run(path, n_up, threshold_ini, threshold_gap, conv_width_ini,
conv_width_gap, roundlo_val_ini, roundlo_gap, roundhi_val_ini,
roundhi_gap, peakmax_val_ini, peakmax_gap):
os.chdir(path)
shift_file = []
file_name = []
x_rms = []
y_rms = []
threshold = []
conv_width = []
roundlo = []
roundhi = []
peakmax = []
for n in range(1, n_up, 1):
outshifts_val = (repr(n) + 'shift.txt')
threshold_val = threshold_ini + (n - 1) * threshold_gap
conv_width_val = conv_width_ini + (n - 1) * conv_width_gap
roundlo_val = roundlo_val_ini + (n - 1) * roundlo_gap
roundhi_val = roundhi_val_ini + (n - 1) * roundhi_gap
peakmax_val = peakmax_val_ini + (n - 1) * peakmax_gap
tweakreg.TweakReg('*c0m.fits',
exclusions='WFPC2_exclude.txt',
updatehdr=False,
shiftfile=True,
outshifts=outshifts_val,
residplot='No plot',
see2dplot=None,
use_sharp_round='Yes',
threshold=threshold_val,
conv_width=conv_width_val,
roundlo=roundlo_val,
roundhi=roundhi_val,
peakmax=peakmax_val)
shift = open(outshifts_val, "r")
list = []
for line in shift:
list.append(line)
shift.close()
for m in range(4, len(list), 1):
words = list[m].split()
print words[1]
if fnmatch.fnmatch(words[1], '*n*'):
shift_file.append(outshifts_val)
file_name.append(words[0])
x_rms.append('failed')
y_rms.append('failed')
threshold.append(threshold_val)
conv_width.append(conv_width_val)
roundlo.append(roundlo_val)
roundhi.append(roundhi_val)
peakmax.append(peakmax_val)
else:
shift_file.append(outshifts_val)
file_name.append(words[0])
x_rms.append(words[5])
y_rms.append(words[6])
threshold.append(threshold_val)
conv_width.append(conv_width_val)
roundlo.append(roundlo_val)
roundhi.append(roundhi_val)
peakmax.append(peakmax_val)
results = table.Table([
shift_file, file_name, x_rms, y_rms, threshold, conv_width, roundlo,
roundhi, peakmax
],
names=[
'shift', 'file', 'x_rms', 'y_rms', 'threshold',
'conv_width', 'roundlo', 'roundhi', 'peakmax'
])
return results
tweakreg.TweakReg(
'*drz.fits', # Pass input images
updatehdr=args.update, # update header with new WCS solution
imagefindcfg={
'threshold': 75,
'cw': cw,
"nbright": nbright_sources
}, # Detection parameters, threshold varies for different data # The object detection threshold above the local background in units of sigma.
refimagefindcfg={
'threshold': 75,
'cw': cw,
"refnbright": ref_bright_sources
}, # Detection parameters, threshold varies for different data # The object detection threshold above the local background in units of sigma.
refcat=ref_cat, # Use user supplied catalog (Gaia)
refimage=refimage,
rfluxcol=5,
rfluxunits="flux",
rminflux=0,
interactive=False,
use_sharp_round=True,
expand_refcat=True,
sharphi=1,
roundhi=1,
roundlo=-1,
sharplo=0.2,
see2dplot=True,
verbose=False,
shiftfile=True, # Save out shift file (so we can look at shifts later)
outshifts='%s_shifts.txt' % wcsname, # name of the shift file
wcsname=wcsname, # Give our WCS a new name
reusename=True,
sigma=3, # sigma clipping parameters
nclip=3,
searchrad=match_radius,
searchunits='arcseconds',
separation=0,
tolerance=1,
minobj=5,
fitgeometry='general',
exclusions=args.mask) # Use the 6 parameter fit
def hst_button(
galaxies,
skymethod='globalmin+match',
instruments="ACS/WFC",
prop_ids=None,
filters=None,
radius=None,
filepath=None,
download_data=True,
correct_astrometry=True,
create_mosaic=True,
jy_conversion=True,
verbose=False,
log_filename='hst.log',
):
"""Create a HST mosaic, given a galaxy name.
Using a galaxy name and radius, queries around that object,
downloads available HST data and mosaics into a final product. It
will create separate mosaics for each proposal ID, and the file structure
will look like ``/galaxy/HST/proposal_id/galaxy_instrument_filter_proposal_id.fits``.
N.B. I must confess to not being well-versed with HST data, so if
anyone can help improve this please let me know.
This data button uses a number of tools included in the drizzlepac
Python package. This includes alignimages/tweakreg and astrodrizzle,
which correct astrometry and are specifically tailored for the setup
of HST data. This means that 1) creating mosaics with this will likely
take a long time and 2) you will need a beefy computer (especially with
regards to hard drive space).
Args:
galaxies (str or list): Names of galaxies to create mosaics for.
Resolved by NED.
skymethod (str, optional): Method used for AstroDrizzle's background
matching step. In general, this can be left untouched but for
mosaics with little overlap, it may be worth playing around
with this. For instance, I've had some luck when there isn't
much overlap between exposures using 'globalmin'. Options are
'localmin', 'globalmin+match', 'globalmin', and 'match'. Defaults
to 'globalmin+match'.
instruments (str or list, optional): Instrument to download data
for. Can be any combination of 'ACS/WFC', 'WFC3/IR',
'WFC3/UVIS', 'WFPC2/PC', or 'WFPC2/WFC'. If you want all
available data for all these instruments, select 'all', but
this is not recommended! Defaults to 'ACS/WFC'.
prop_ids (str or list, optional): Proposal IDs to download data for.
Defaults to None, which will pull out all proposal IDs for each
instrument.
filters (str or list, optional): Filters to download data for.
The script will look for each filter, for each instrument.
Defaults to None, which will pull out all applicable filters
for each instrument, for each proposal ID.
radius (astropy.units.Quantity, optional): Radius around the
galaxy to search for observations. Defaults to None, where
it will query Ned to get size.
filepath (str, optional): Path to save the working and output
files to. If not specified, saves to current working
directory.
download_data (bool, optional): If True, will download data from
MAST. Defaults to True.
correct_astrometry (bool, optional): If True, will perform astrometric
corrections to the downloaded data using alignimages. Defaults
to True.
create_mosaic (bool, optional): Switching this to True will
mosaic the data using astrodrizzle as appropriate. Defaults
to True.
jy_conversion (bool, optional): Convert the mosaicked file from
raw units to Jy/pix. Defaults to True.
verbose (bool, optional): Can be used to suppress most of the
output messages produced during the process. Mainly useful
for debugging. Defaults to False.
log_filename (str, optional): Will produce a stripped down log
of what data the code is reducing. By default, will save to
galaxy/hst.log.
"""
if isinstance(galaxies, str):
galaxies = [galaxies]
if isinstance(instruments,str):
instruments = [instruments]
if instruments == 'all':
instruments = ['ACS/WFC',
'WFC3/IR','WFC3/UVIS',
'WFPC2/PC','WFPC2/WFC']
if isinstance(filters,str):
filters = [filters]
if isinstance(prop_ids,str):
prop_ids = [prop_ids]
if filepath is not None:
os.chdir(filepath)
orig_dir = os.getcwd()
if radius is not None:
original_radius = radius.copy()
else:
original_radius = None
steps = []
if download_data:
steps.append(1)
if correct_astrometry:
steps.append(2)
if create_mosaic:
steps.append(3)
if jy_conversion:
steps.append(4)
# Set up folders for various corrections
os.environ['CRDS_SERVER_URL'] = 'https://hst-crds.stsci.edu'
os.environ['CRDS_PATH'] = orig_dir+'/reference_files'
os.environ['iref'] = orig_dir+'/reference_files/references/hst/wfc3/'
os.environ['jref'] = orig_dir+'/reference_files/references/hst/acs/'
os.environ['uref'] = orig_dir+'/reference_files/references/hst/wfpc2/'
# For large proposals, astrodrizzle can run into file open
# issues so raise the max file open amount.
_, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE,(hard,hard))
# Change the temp directory -- if this gets filled up it can cause
# problems.
orig_tmpdir = os.environ['TMPDIR']
if not os.path.exists('tmp'):
os.mkdir('tmp')
os.environ['TMPDIR'] = orig_dir+'/tmp'
for galaxy in galaxies:
if not os.path.exists(galaxy):
os.mkdir(galaxy)
if not os.path.exists(galaxy+'/HST'):
os.mkdir(galaxy+'/HST')
if not verbose:
# Various packages used here put out a lot of messages. Silence info messages.
loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
for logger in loggers:
logger.setLevel(logging.ERROR)
# Even if verbose is not True, still print out some useful messages to the
# console.
hst_logger = logging.getLogger('data_buttons')
handler = logging.FileHandler(galaxy+'/'+log_filename,mode='w')
hst_logger.addHandler(handler)
hst_logger.addHandler(logging.StreamHandler())
hst_logger.setLevel(logging.INFO)
hst_logger.info('Beginning '+galaxy)
hst_logger.info(' ')
hst_logger.info(' ')
if radius is None:
try:
size_query = Ned.get_table(galaxy,table='diameters')
radius = np.max(size_query['NED Major Axis'])/2*u.arcsec
radius = radius.to(u.deg)
except:
hst_logger.warning(galaxy+' not resolved by Ned, using 0.2deg radius.')
radius = 0.2*u.degree
obs_table = Observations.query_criteria(objectname=galaxy,
radius=radius,
obs_type='all',
obs_collection='HST')
# Ignore any calibration observations.
obs_table = obs_table[obs_table['intentType'] == 'science']
for instrument in instruments:
# Pixel sizes for final mosaics selected to match the HLA.
pix_size = {'ACS/HRC':0.025,
'ACS/SBC':0.03,
'ACS/WFC':0.05,
'NICMOS/NIC1':0.025,
'NICMOS/NIC2':0.05,
'NICMOS/NIC3':0.1,
'WFC3/IR':0.09,
'WFC3/UVIS':0.04,
'WFPC2/PC':0.05,
'WFPC2/WFC':0.1}[instrument]
# Bits to consider good for drizzling.
bits = {'ACS/HRC':256,
'ACS/SBC':256,
'ACS/WFC':256,
'NICMOS/NIC1':0,
'NICMOS/NIC2':0,
'NICMOS/NIC3':0,
'WFC3/IR':768,
'WFC3/UVIS':256,
'WFPC2/PC':'8,1024',
'WFPC2/WFC':'8,1024'}[instrument]
# Filename extension, in order of preference.
suffixes = {'ACS/WFC':['FLC','FLT'],
'WFC3/IR':['FLT'],
'WFC3/UVIS':['FLC','FLT'],
'WFPC2/PC':[['C0M','C1M']],
'WFPC2/WFC':[['C0M','C1M']],
}[instrument]
# The instruments often have / in the name, so account for
# this in making folders and files.
hst_logger.info('Beginning '+instrument)
if not os.path.exists(galaxy+'/HST/'+instrument.replace('/','_')):
os.mkdir(galaxy+'/HST/'+instrument.replace('/','_'))
reset_filters = False
instrument_table = obs_table[obs_table['instrument_name'] == instrument]
reset_prop_ids = False
if not prop_ids:
prop_ids = list(np.unique(instrument_table['proposal_id']))
reset_prop_ids = True
hst_logger.info('Available proposal IDs: '+','.join(prop_ids))
hst_logger.info(' ')
for prop_id in prop_ids:
hst_logger.info('Proposal ID: '+str(prop_id))
prop_table = instrument_table[instrument_table['proposal_id'] == prop_id]
if not filters:
filters = list(np.unique(prop_table['filters']))
reset_filters = True
hst_logger.info('Available filters: '+','.join(filters))
for hst_filter in filters:
# If we have a highly illegal filter, just skip.
# TODO: This needs to be sorted for some fringe
# cases, probably.
if not hst_filter[0] == 'F':
continue
hst_logger.info('Filter: '+str(hst_filter))
# Pull out available data and download.
filter_table = prop_table[prop_table['filters'] == hst_filter]
if len(filter_table) == 0:
hst_logger.warning('No available data to download. Skipping...')
continue
data_products_id = Observations.get_product_list(filter_table)
for suffix in suffixes:
download_table = Observations.filter_products(data_products_id,
productSubGroupDescription=suffix,
mrp_only=False)
if len(download_table) > 0:
break
if isinstance(suffix,list):
filename_exts = [ext.lower() for ext in suffix]
else:
filename_exts = [suffix.lower()]
hst_logger.info(instrument+'/'+prop_id+'/'+hst_filter)
if not os.path.exists(galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter):
os.mkdir(galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter)
if not os.path.exists(galaxy+'/HST/'+prop_id):
os.mkdir(galaxy+'/HST/'+prop_id)
full_filepath = (galaxy+
'/HST/'+
instrument.replace('/','_')+
'/'+
hst_filter+
'/'
+prop_id)
if not os.path.exists(full_filepath):
os.mkdir(full_filepath)
if 1 in steps:
# Download files
download_mast(download_table,
download_dir="hst_temp/" + galaxy)
if not os.path.exists(full_filepath+'/raw'):
os.mkdir(full_filepath+'/raw')
if not os.path.exists(full_filepath+'/outputs'):
os.mkdir(full_filepath+'/outputs')
# Pull out the relevant files, and move to base folder.
for filename_ext in filename_exts:
matches = []
for root, _, filenames in os.walk("hst_temp/" + galaxy):
for filename in fnmatch.filter(
filenames, "*_"+filename_ext+".fits"
):
matches.append(os.path.join(root, filename))
for match in matches:
filename = match.split('/')
os.rename(match,full_filepath+'/raw/'+filename[-1])
# Clean up any temporary files.
shutil.rmtree("hst_temp/" + galaxy, ignore_errors=True)
filename_ext = filename_exts[0]
hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits')
if 2 in steps:
# First, update the WCS information in case it's
# required.
for filename_ext in filename_exts:
hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits')
crds.assign_bestrefs(hst_files,
sync_references=True)
# For WFPC2, the CRDS doesn't download everything
# needed. Download the GEIS data files and
# rerun the bestrefs assignment.
if 'WFPC2' in instrument:
geis_hdrs = glob.glob(os.environ['uref']+'/*h')
for geis_hdr in geis_hdrs:
geis_data = geis_hdr[:-1]+'d'
if not os.path.exists(geis_data):
geis_data = geis_data.split('/')[-1]
print(geis_data)
print(os.environ['uref'])
wget.download(
os.environ['CRDS_SERVER_URL']+'/unchecked_get/references/hst/'+geis_data,
out=os.environ['uref'])
crds.assign_bestrefs(hst_files,sync_references=True)
for hst_file in hst_files:
stwcs.updatewcs.updatewcs(hst_file,
use_db=False)
os.chdir(full_filepath+'/raw')
filename_ext = filename_exts[0]
hst_files = glob.glob('*_'+filename_ext+'.fits')
# Normalize all files.
photeq.photeq(', '.join(hst_files),readonly=False)
os.rename('photeq.log','../outputs/photeq.log')
if 'WFPC' in instrument:
# Using tweakreg, align each frame to GAIA.
gaia_table = Gaia.query_object_async(coordinate=galaxy,
radius=2*radius)
ras = gaia_table['ra']
decs = gaia_table['dec']
source_table = Table([ras,decs])
source_table.write('gaia.cat',
format='ascii.fast_commented_header')
tweakreg.TweakReg(hst_files,
imagefindcfg={'threshold':5,'conv_width':3},
refcat='gaia.cat',
#expand_refcat=True,
enforce_user_order=False,
shiftfile=True,
outshifts='shifts.txt',
searchrad=10,
minobj=5,
separation=0,
updatehdr=True,
reusename=True,
wcsname='TWEAK',
interactive=False,
fitgeometry='general',
clean=True,
see2dplot=False
)
# Update the c1m files to use the TWEAK
# wcs
for hst_file in hst_files:
dq_file = hst_file.replace('c0','c1')
tweakback.tweakback(hst_file,
dq_file,
newname='TWEAK')
plot_files = glob.glob('*.png')
for plot_file in plot_files:
os.remove(plot_file)
cat_files = glob.glob('*.coo')
for cat_file in cat_files:
os.remove(cat_file)
os.rename('shifts_wcs.fits','../outputs/shifts_wcs.fits')
os.rename('tweakreg.log','../outputs/tweakreg.log')
os.rename('shifts.txt','../outputs/shifts.txt')
elif 'ACS' in instrument or 'WFC3' in instrument:
# Correct astrometry using alignimages. First,
# correct each frame separately.
pool = mp.Pool(mp.cpu_count())
suitable_hst_files = pool.map(astrometric_correction,
hst_files)
pool.close()
suitable_hst_files = [x for x in suitable_hst_files
if x is not None]
if len(suitable_hst_files) == 0:
hst_logger.warning('Failure with astrometry corrections. Skipping')
os.chdir(orig_dir)
continue
# Now, align every suitable frame simultaneously.
output_table = astrometric_correction(suitable_hst_files)
with open('../outputs/astrometry.pkl','wb') as table_file:
pickle.dump(output_table,table_file)
else:
raise Exception('Unknown instrument!')
os.chdir(orig_dir)
os.chdir(full_filepath)
if 3 in steps:
os.chdir('raw')
if 'WFPC2' in instrument:
hst_files = glob.glob('*_c0m.fits')
wcskey = 'TWEAK'
elif 'ACS' in instrument or 'WFC3' in instrument:
with open('../outputs/astrometry.pkl','rb') as table_file:
output_table = pickle.load(table_file)
# We only want fits where an acceptable astrometric
# solution has been found.
suitable_fits = np.where(output_table['fit_qual'] < 5)
# (output_table['fit_qual'] >= 1)
hst_files = list(output_table[suitable_fits]['imageName'])
if len(output_table[suitable_fits]) == 0:
hst_logger.warning('Failure with astrometry corrections. Skipping')
os.chdir(orig_dir)
continue
wcskey = ' '
else:
raise Exception('Unknown instrument!')
# Following Dalcanton+ (2012), group exposures into
# long (>50s) and short (<=50s), and process for cosmic
# rays separately
exp_times = []
for hst_file in hst_files:
hdu = fits.open(hst_file)[0]
exp_time = hdu.header['EXPTIME']
exp_times.append(exp_time)
for exp_group in ['short','long']:
hst_files_group = []
for i in range(len(exp_times)):
if exp_times[i] > 50 and exp_group == 'long':
hst_files_group.append(hst_files[i])
elif exp_times[i] <= 50 and exp_group == 'short':
hst_files_group.append(hst_files[i])
if len(hst_files_group) == len(hst_files):
exp_group = ''
if len(hst_files_group) == 0:
continue
if len(exp_group) > 0:
output_name = '../outputs/'+galaxy+'_'+exp_group
drizzle_log_name = '../outputs/astrodrizzle_'+exp_group+'.log'
else:
output_name = '../outputs/'+galaxy
drizzle_log_name = '../outputs/astrodrizzle.log'
# Perform the mosaicking. Generally, use iminmed.
# However, sometimes iminmed will fail so
# for the other instruments we'll use imedian as
# a fallback.
combine_types = ['iminmed','imedian']
if 'WFPC2' in instrument:
combine_nhigh = 1
else:
combine_nhigh = 0
for combine_type in combine_types:
try:
astrodrizzle.AstroDrizzle(
input=hst_files_group,
output=output_name,
preserve=False,
clean=True,
combine_type=combine_type,
combine_nhigh=combine_nhigh,
skymethod=skymethod,
sky_bits=bits,
driz_sep_bits=bits,
driz_sep_fillval=99999,
combine_hthresh=90000,
final_scale=pix_size,
final_bits=bits,
final_fillval=0,
wcskey=wcskey,
final_rot=0,
)
break
except ValueError:
pass
# Move the AstroDrizzle log.
os.rename('astrodrizzle.log',
drizzle_log_name)
# Move back to the original directory.
os.chdir(orig_dir)
if 4 in steps:
mosaic_outputs = glob.glob(full_filepath+'/outputs/*_sci.fits')
for mosaic_output in mosaic_outputs:
# Replace any fillvals with NaNs.
hdu = fits.open(mosaic_output)[0]
hdu.data[hdu.data == 0] = np.nan
fits.writeto(mosaic_output,
hdu.data,hdu.header,
overwrite=True)
if '_long_' in mosaic_output.split('/')[-1]:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'_long.fits')
elif '_short_' in mosaic_output.split('/')[-1]:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'_short.fits')
else:
new_filename = (galaxy+
'/HST/'
+prop_id
+'/'
+galaxy
+'_'
+instrument.replace('/','_')
+'_'
+hst_filter
+'_'
+prop_id
+'.fits')
convert_to_jy(mosaic_output,
new_filename)
if reset_filters:
filters = None
hst_logger.info(' ')
if reset_prop_ids:
prop_ids = None
hst_logger.info(' ')
if original_radius is None:
radius = None
else:
radius = original_radius.copy()
# Clear out the tmp folder and reset to the original.
shutil.rmtree('tmp/', ignore_errors=True)
os.environ['TMPDIR'] = orig_tmpdir
def RunTweakReg(
files='*fl?.fits',
refcat=None,
refim=None,
wcsname='SNDRIZPIPE',
refnbright=None,
rmaxflux=None,
rminflux=None,
searchrad=1.0,
# fluxmax=None, fluxmin=None,
nclip=3,
sigmaclip=3.0,
computesig=False,
peakmin=None,
peakmax=None,
threshold=4.0,
minobj=10,
nbright=None,
fitgeometry='rscale',
interactive=False,
clean=False,
clobber=False,
verbose=False,
debug=False):
"""Run tweakreg on a set of flt files or drz files."""
if debug:
import pdb
pdb.set_trace()
from astropy.io import ascii
#convert the input list into a useable list of images for astrodrizzle
if type(files) == str:
filelist = tools.parseinput.parseinput(files)[0]
else:
filelist = files
files = ','.join(filelist).strip(',')
if len(filelist) == 0:
raise RuntimeError("List of input files has no real files: %s" %
str(files))
# pre-clean any existing WCS header info
if clobber:
clearAltWCS(filelist)
hdr1 = pyfits.getheader(filelist[0])
conv_width = getconvwidth(filelist[0])
if computesig == False:
skysigma = getskysigma(filelist)
else:
skysigma = 0.0
if (nbright and nbright < 3) or minobj < 3:
use2dhist = False
else:
use2dhist = True
# check first if the WCS wcsname already exists in the first image
wcsnamelist = [
hdr1[k] for k in list(hdr1.keys()) if k.startswith('WCSNAME')
]
if wcsname in wcsnamelist and not clobber:
print(("WCSNAME %s already exists in %s" % (wcsname, filelist[0]) +
"so I'm skipping over this tweakreg step." +
"Re-run with clobber=True if you really want it done."))
return (wcsname)
rfluxcol, rfluxunits = None, None
if refcat:
refCatalog = ascii.read(refcat)
if len(refCatalog.columns) > 2:
for icol in range(len(refCatalog.columns)):
if 'flux' in refCatalog.colnames[icol].lower():
rfluxcol = icol + 1
rfluxunits = 'flux'
elif 'mag' in refCatalog.colnames[icol].lower():
rfluxcol = icol + 1
rfluxunits = 'mag'
if not refim:
refim = filelist[0]
if interactive:
while True:
if nbright:
# make source catalogs for each SCI extension
srcCatListFile = mkSourceCatList(filelist,
filelist[0].split('_')[0] +
'_cat.list',
threshold=threshold,
peakmin=peakmin,
peakmax=peakmax)
if verbose:
print(("Generated a list of source catalogs : %s" %
srcCatListFile))
xcol, ycol, fluxcol = 1, 2, 3
else:
srcCatListFile = None
nbright = None
xcol, ycol, fluxcol = 1, 2, None
print(
"sndrizzle.register: Running a tweakreg loop interactively.")
tweakreg.TweakReg(
files,
updatehdr=False,
wcsname='TWEAK',
use2dhist=use2dhist,
see2dplot=True,
residplot='both',
fitgeometry=fitgeometry,
nclip=nclip,
computesig=computesig,
skysigma=skysigma,
sigma=sigmaclip,
refcat=refcat,
refimage=refim,
refxcol=1,
refycol=2,
refxyunits='degrees',
rfluxcol=rfluxcol,
rfluxunits=rfluxunits,
refnbright=refnbright,
rmaxflux=rmaxflux,
rminflux=rminflux,
# fluxmax=fluxmax, fluxmin=fluxmin,
peakmax=peakmax,
peakmin=peakmin,
searchrad=searchrad,
conv_width=conv_width,
threshold=threshold,
separation=0,
tolerance=1.0,
minobj=minobj,
catfile=srcCatListFile,
nbright=nbright,
xcol=xcol,
ycol=ycol,
fluxcol=fluxcol,
fluxunits='cps',
clean=(clean and not nbright),
writecat=(not nbright))
print("==============================\n sndrizzle.register:\n")
userin = input("Adopt these tweakreg settings? y/[n]").lower()
if userin.startswith('y'):
print("OK. Proceeding to update headers.")
break
print("Current tweakreg/imagefind parameters:")
printfloat(
" refnbright = %i # number of brightest refcat sources to use",
refnbright)
printfloat(" rminflux = %.1f # min flux for refcat sources",
rminflux)
printfloat(" rminflux = %.1f # max flux for refcat sources",
rmaxflux)
printfloat(
" searchrad = %.1f # matching search radius (arcsec)",
searchrad)
printfloat(
" peakmin = %.1f # min peak flux for detected sources",
peakmin)
printfloat(
" peakmax = %.1f # max peak flux for detected sources",
peakmax)
# printfloat(" fluxmin = %.1f # min total flux for detected sources", fluxmin )
# printfloat(" fluxmax = %.1f # max total flux for detected sources", fluxmax )
printfloat(" threshold = %.1f # detection threshold in sigma ",
threshold)
printfloat(
" fitgeometry= %s # fitting geometry [shift,rscale] ",
fitgeometry)
printfloat(
" nclip = %i # Number of sigma-clipping iterations",
nclip)
printfloat(" sigmaclip = %.1f # Clipping limit in sigmas ",
sigmaclip)
print('Adjust parameters using "parname = value" syntax.')
print('Enter "run" to re-run tweakreg with new parameters.')
while True:
userin = input(" ").lower()
if userin.startswith('run'): break
try:
parname = userin.split('=')[0].strip()
value = userin.split('=')[1].strip()
except:
print('Must use the "parname = value" syntax. Try again')
continue
if parname == 'refnbright': refnbright = int(value)
elif parname == 'rminflux': rminflux = float(value)
elif parname == 'rmaxflux': rmaxflux = float(value)
elif parname == 'searchrad': searchrad = float(value)
elif parname == 'peakmin': peakmin = float(value)
elif parname == 'peakmax':
peakmax = float(value)
# elif parname=='fluxmax' : fluxmax=float( value )
# elif parname=='fluxmin' : fluxmin=float( value )
elif parname == 'threshold':
threshold = float(value)
elif parname == 'fitgeometry':
fitgeometry = value
elif parname == 'nclip':
nclip = int(value)
elif parname == 'sigmaclip':
sigmaclip = float(value)
print("==============================\n sndrizzle.register:\n")
print(" Final tweakreg run for updating headers.")
if nbright:
# make source catalogs for each SCI extension
srcCatListFile = mkSourceCatList(filelist,
filelist[0].split('_')[0] +
'_cat.list',
threshold=threshold,
peakmin=peakmin,
peakmax=peakmax)
if verbose:
print(
("Generated a list of source catalogs : %s" % srcCatListFile))
xcol, ycol, fluxcol = 1, 2, 3
else:
srcCatListFile = None
nbright = None
xcol, ycol, fluxcol = 1, 2, None
print(('fitgeometry = %s' % fitgeometry))
tweakreg.TweakReg(
files,
updatehdr=True,
wcsname=wcsname,
use2dhist=use2dhist,
see2dplot=False,
residplot='No plot',
computesig=computesig,
skysigma=skysigma,
fitgeometry=fitgeometry,
nclip=nclip,
sigma=sigmaclip,
refcat=refcat,
refimage=refim,
refxcol=1,
refycol=2,
refxyunits='degrees',
refnbright=refnbright,
rfluxcol=rfluxcol,
rfluxunits=rfluxunits,
rmaxflux=rmaxflux,
rminflux=rminflux,
# fluxmax=fluxmax, fluxmin=fluxmin,
peakmax=peakmax,
peakmin=peakmin,
searchrad=searchrad,
conv_width=conv_width,
threshold=threshold,
catfile=srcCatListFile,
nbright=nbright,
xcol=xcol,
ycol=ycol,
fluxcol=fluxcol,
fluxunits='cps',
separation=0,
tolerance=1.0,
minobj=minobj,
clean=clean,
writecat=not nbright)
return (wcsname)
def offset_position():
"""
Check grism extractions for 13580 program that put Vy2-2 just
off of the left edge of the frame to test extrapolation of wavelength
calibration.
"""
import astropy.io.fits as pyfits
from astropy.table import Table as table
import drizzlepac
from drizzlepac import tweakreg, tweakback
import stwcs
import unicorn
unicorn.candels.make_asn_files(uniquename=True)
info = table.read('files.info', format='ascii.commented_header')
for filter in ['F098M', 'F105W']:
filter_files = list(info['FILE'][info['FILTER'] == filter])
#
files = glob.glob('VY2-2*%s_asn.fits' %(filter))
for file in files:
prep.prep_direct_grism_pair(direct_asn=file, grism_asn=False, radec='2mass.radec', scattered_light=False, skip_direct=False)
#
driz_images = glob.glob('VY2-2*%s_drz_sci.fits' %(filter))
tweakreg.TweakReg(driz_images, refimage=driz_images[0], updatehdr=True, updatewcs=True, catfile=None, xcol=2, ycol=3, xyunits='pixels', refcat=None, refxcol=1, refycol=2, refxyunits='degrees', shiftfile=True, outshifts='%s_shifts.txt' %(filter), outwcs='%s_wcs.fits' %(filter), searchrad=5, tolerance=12, wcsname='TWEAK', interactive=False, residplot='No plot', see2dplot=False, clean=True, headerlet=True, clobber=True)
tweakback.tweakback(driz_images[1])
#
drizzlepac.astrodrizzle.AstroDrizzle(filter_files, output='VY22-%s' %(filter), clean=True, skysub=False, final_scale=None, final_pixfrac=1, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7')
drizzlepac.astrodrizzle.AstroDrizzle(filter_files, output='VY22-%s' %(filter), clean=True, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True)
### Put WCS from direct F105W images into G102 at same POS-TARG
info = table.read('files.info', format='ascii.commented_header')
idx = np.arange(len(info))[info['FILTER'] == 'F105W']
asn = threedhst.utils.ASNFile('../RAW/ibhj01030_asn.fits')
for i in idx:
direct = info['FILE'][i]
dx, dy = info['POSTARG1'][i], info['POSTARG2'][i]
ix_gris = (info['POSTARG1'] == dx) & (info['POSTARG2'] == dy) & (info['FILTER'] == 'G102')
grism = info['FILE'][ix_gris][0]
sign = {True:'+', False:'-'}
#
asn.product = 'VY22%s%02d%s%02d-F105W' %(sign[dx > 0], np.abs(dx), sign[dy > 0], np.abs(dy))
asn.exposures = [direct.split('_flt')[0]]
asn.write(asn.product + '_asn.fits')
#
asn.product = 'VY22%s%02d%s%02d-G102' %(sign[dx > 0], np.abs(dx), sign[dy > 0], np.abs(dy))
asn.exposures = [grism.split('_flt')[0]]
asn.write(asn.product + '_asn.fits')
#### update WCS header
imd = pyfits.open(direct)
img = pyfits.open(grism)
sci_ext=1
direct_WCS = stwcs.wcsutil.HSTWCS(imd, ext=sci_ext)
drizzlepac.updatehdr.update_wcs(grism, sci_ext, direct_WCS, verbose=True)
#### Make reference catalog
root = 'VY22-F105W'
se = threedhst.sex.SExtractor()
se.aXeParams()
se.copyConvFile()
se.overwrite = True
se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION, BACKGROUND'
se.options['CHECKIMAGE_NAME'] = '%s_drz_seg.fits, %s_drz_bkg.fits' %(root, root)
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
se.options['WEIGHT_IMAGE'] = '%s_drz_wht.fits' %(root)
se.options['WEIGHT_GAIN'] = 'Y'
se.options['GAIN'] = '0'
se.options['FILTER'] = 'Y'
se.options['DETECT_THRESH'] = '2.'
se.options['ANALYSIS_THRESH'] = '2.'
se.options['DETECT_MINAREA'] = '10'
se.options['MASK_TYPE'] = 'NONE'
se.options['DEBLEND_NTHRESH'] = '64'
se.options['DEBLEND_MINCONT'] = '0.1'
se.options['SEEING_FWHM'] = '0.12'
se.options['BACK_TYPE'] = 'MANUAL'
se.options['BACKPHOTO_TYPE'] = 'LOCAL'
se.options['MAG_ZEROPOINT'] = '%.2f' %(unicorn.reduce.ZPs['F105W'])
se.options['CATALOG_TYPE'] = 'ASCII_HEAD'
se.options['CATALOG_NAME'] = '%s_drz_sci.cat' %(root)
status = se.sextractImage('%s_drz_sci.fits[0]' %(root))
threedhst.sex.sexcatRegions('%s_drz_sci.cat' %(root), '%s_drz_sci.reg' %(root), format=1)
#### Make interlaced images
files = glob.glob('VY22[+-]??[+-]??-F105W_asn.fits')
for file in files:
unicorn.reduce.interlace_combine(file.split('_asn')[0], growx=1, growy=1, NGROW=50, pad=60, view=False)
unicorn.reduce.interlace_combine(file.split('_asn')[0].replace('F105W', 'G102'), growx=1, growy=1, NGROW=50, pad=60, view=False)
red = unicorn.reduce
red.adriz_blot_from_reference(pointing=file.split('_asn')[0], pad=60, NGROW=50, growx=1, growy=1, auto_offsets=False, ref_exp=0, ref_image='VY22-F105W_drz_sci.fits', ref_ext=0, ref_filter='F105W', seg_image='VY22-F105W_drz_seg.fits', cat_file='VY22-F105W_drz_sci.cat')
### extract spectra
id = 798
files = glob.glob('VY22[+-]??[+-]??-F105W_asn.fits')
for file in files:
model = unicorn.reduce.GrismModel(root=file.split('-F10')[0], direct='F105W', grism='G102', growx=1, growy=1, grow_factor=1)
model.twod_spectrum(id, miny=-30, refine=False, CONTAMINATING_MAGLIMIT=0)
files = glob.glob('*2D.fits')
yi, xi = np.indices((30,30))
xs, ys = np.zeros(len(files)), np.zeros(len(files))
xpix = xs*0
for i, file in enumerate(files):
twod = unicorn.reduce.Interlace2D(file)
xs[i] = np.sum(xi*twod.im['DSCI'].data/twod.im['DSCI'].data.sum())
ys[i] = np.sum(xi*twod.im['DSCI'].data/twod.im['DSCI'].data.sum())
xpix[i] = twod.im[0].header['X_PIX']
xs -= np.median(xs) #+ 0.5
ys -= np.median(ys)
#xs -= 0.5
fig = plt.figure(figsize=[16,4])
fig.subplots_adjust(left=0.04, right=0.98, top=0.92)
for i, file in enumerate(files):
twod = unicorn.reduce.Interlace2D(file)
w, f = twod.optimal_extract(twod.im['SCI'].data)
c = {True: 'red', False: 'blue'}
#plt.plot(w-np.diff(w)[0]*xs[i], f/twod.im['SENS'].data, alpha=0.5, marker='o', ms=2, label='%s, %s' %(file[4:7], file[7:10])) # , color=c['2-2' in file]
ff = f*0.
for k in range(ff.shape[0]):
y0 = int(np.round(twod.im['YTRACE'].data[k]))
ff[k] = np.sum(twod.im['SCI'].data[y0-4:y0+4, k])
#
plt.plot(w-np.diff(w)[0]*xs[i], ff/twod.im['SENS'].data, alpha=0.5, marker='o', ms=2, label='%s, %s' %(file[4:7], file[7:10])) # , color=c['2-2' in file]
#plt.plot(twod.oned.data['wave']-np.diff(w)[0]*xs[i], twod.oned.data['flux']/twod.oned.data['sensitivity'], alpha=0.5, marker='o', ms=2, label='%s, %s' %(file[4:7], file[7:10])) # , color=c['2-2' in file]
#
print file, np.diff(w)[0]
#ds9.frame(i+1)
#ds9.view(twod.im['DSCI'].data)
PNe_lines = [9071.403457, 9534.921052, 10049.850283, 10833.000000, 12821.000000, 16112.000000, 16412.000000]
for line in PNe_lines:
plt.plot([line, line], [0.1,1.e5], color='black', linewidth=3, alpha=0.2, zorder=-5)
#plt.plot(w-np.diff(w)[0]*xs[i]-np.diff(w)[0], f/twod.im['SENS'].data, alpha=0.5, color='green', marker='o', ms=2)
plt.legend(loc='upper right', prop={'size':9}, title='POS-TARG')
plt.title('VY2-2, G102, 13580')
plt.xlim(8500, 11500)
plt.ylim(700,14000)
plt.ylim(600,64000)
plt.semilogy()
plt.xlabel(r'$\lambda$')
plt.savefig('vy22-edge_v2.pdf') #, dpi=100)
plt.close()
pkmax = 70000.
pkmin = 1000.
else:
print 'Cannot figure out camera for {}'.format(im)
continue
root = im[:10]
tweak.TweakReg(im,\
writecat=True,\
clean=True,\
runfile='test.log',\
conv_width=conv_w,\
updatehdr=False,\
headerlet=False,\
shiftfile=False,\
catfile='',\
computesig=True,\
peakmax=pkmax,\
peakmin=pkmin,\
threshold=10.)
columns = hdr1['naxis1']
rows = hdr1['naxis2']
redge = columns - 50
ledge = 50
tedge = rows - 50
bedge = 50
def findImprovedAlignment(self):
"""
Tries to find stars to be used for improved alignment.
Generates coordinate lists for each POL file and for every _flt file.
Maps all the positions to uncorrected/distorted frame using pixtopix transformation.
Finally, runs the tweakreg to find improved alignment and updates the WCS in the
headers.
"""
#find some good stars
source = sourceFinding(pf.open(self.settings['sourceImage'])[1].data)
results = source.runAll()
#find improved locations for each star
acc = []
for x, y in zip(results['xcms'], results['ycms']):
acc.append(
iraf.imcntr('POL*_drz.fits[1]',
x_init=x,
y_init=y,
cboxsize=45,
Stdout=1))
o = open('tmp.txt', 'w')
o.write('#File written on {0:>s}\n'.format(
datetime.datetime.isoformat(datetime.datetime.now())))
for line in acc:
for l in line:
o.write(l.replace('[', '').replace(']', '') + '\n')
o.close()
data = open('tmp.txt').readlines()
pol0 = open('POL0coords.txt', 'w')
pol60 = open('POL60coords.txt', 'w')
pol120 = open('POL120coords.txt', 'w')
pol0r = open('POL0coords.reg', 'w')
pol60r = open('POL60coords.reg', 'w')
pol120r = open('POL120coords.reg', 'w')
for line in data:
tmp = line.split(':')
x = tmp[1].replace('y', '').strip()
y = tmp[2].strip()
out = '%s %s\n' % (x, y)
reg = 'image;circle(%s,%s,5)\n' % (x, y)
if 'POL0' in line:
pol0.write(out)
pol0r.write(reg)
elif 'POL60' in line:
pol60.write(out)
pol60r.write(reg)
elif 'POL120' in line:
pol120.write(out)
pol120r.write(reg)
else:
print 'Skipping line:', line
pol0.close()
pol60.close()
pol120.close()
pol0r.close()
pol60r.close()
pol120r.close()
data = open('../rawFiles.txt').readlines()
pol0 = [
line.split()[0].split('_raw')[0] + '_flt.fits' for line in data
if 'POL0' in line.split()[2]
]
pol60 = [
line.split()[0].split('_raw')[0] + '_flt.fits' for line in data
if 'POL60' in line.split()[2]
]
pol120 = [
line.split()[0].split('_raw')[0] + '_flt.fits' for line in data
if 'POL120' in line.split()[2]
]
for file in pol0:
x, y = pixtopix.tran(file + "[1]",
'POL0V_drz.fits[1]',
'backward',
coords='POL0coords.txt',
output=file.replace('.fits', '') + '.coords',
verbose=False)
for file in pol60:
x, y = pixtopix.tran(file + "[1]",
'POL60V_drz.fits[1]',
'backward',
coords='POL60coords.txt',
output=file.replace('.fits', '') + '.coords',
verbose=False)
for file in pol120:
x, y = pixtopix.tran(file + "[1]",
'POL120V_drz.fits[1]',
'backward',
coords='POL120coords.txt',
output=file.replace('.fits', '') + '.coords',
verbose=False)
del x
del y
coords = glob.glob('*_flt.coords')
#remove comment lines from each coords file and produce a DS9 region file
for f in coords:
data = open(f).readlines()
out = open(f, 'w')
reg = open(f.replace('.coords', '.reg'), 'w')
reg.write('#File written on {0:>s}\n'.format(
datetime.datetime.isoformat(datetime.datetime.now())))
for line in data:
if not line.startswith('#'):
out.write(line)
tmp = line.split()
reg.write('image;circle({0:>s},{1:>s},5)\n'.format(
tmp[0], tmp[1]))
out.close()
reg.close()
#create a mapping file
out = open('regcatalog.txt', 'w')
for f in coords:
out.write('%s %s\n' % (f.replace('.coords', '.fits'), f))
out.close()
params = {
'catfile': 'regcatalog.txt',
'shiftfile': True,
'outshifts': 'flt1_shifts.txt',
'updatehdr': True,
'verbose': False,
'minobj': 15,
'use2dhist': False,
'see2dplot': False,
'searchrad': 50,
'searchunits': 'pixels',
'tolerance': 50.0,
'separation': 30.0,
'nclip': 3
}
tweakreg.TweakReg('*_flt.fits', editpars=False, **params)
params.update({
'outshifts': 'flt_shifts.txt',
'searchrad': 15,
'tolerance': 3
})
tweakreg.TweakReg('*_flt.fits', editpars=False, **params)
elif detector == 'ir':
input_images = sorted(glob.glob('i*flt.fits'))
threshold = 20.
cw = 2.5
# Parallelized option
p = Pool(8)
derp = p.map(updatewcs.updatewcs, input_images)
p.close()
p.join()
cat = 'gaia.cat'
wcsname ='GAIA'
teal.unlearn('tweakreg')
teal.unlearn('imagefindpars')
tweakreg.TweakReg(input_images, # Pass input images
updatehdr=True, # update header with new WCS solution
imagefindcfg={'threshold':threshold,'conv_width':cw},# Detection parameters, threshold varies for different data
separation=0., # Allow for very small shifts
refcat=cat, # Use user supplied catalog (Gaia)
clean=True, # Get rid of intermediate files
interactive=False,
see2dplot=True,
shiftfile=True, # Save out shift file (so we can look at shifts later)
wcsname=wcsname, # Give our WCS a new name
reusename=True,
fitgeometry='general') # Use the 6 parameter fit
os.rename('shifts.txt','shifts_{}.txt'.format(detector))
os.rename('shifts_wcs.fits','shifts_{}_wcs.fits'.format(detector))
def registerPOLs(self):
"""
Aligns and registers the POL files. In addition, produces block averages files.
:return: None
"""
#copy the first round files to backup and check the shifts
drzs = glob.glob('*POL*_drz*.fits')
for drz in drzs:
shutil.move(drz, drz.replace('_drz.fits', '_backup.fits'))
params = {
'outshifts': 'backup_shifts.txt',
'updatehdr': True,
'verbose': False,
'minobj': 20,
'use2dhist': True,
'residplot': 'residuals',
'see2dplot': False,
'searchrad': 35,
'searchunits': 'pixels'
}
#we can do this twice to get the alignment really good
tweakreg.TweakReg('*_backup.fits', editpars=False, **params)
tweakreg.TweakReg('*_backup.fits', editpars=False, **params)
#user sregister to resample to separate POL frames to a single WCS frame
sregister('POL*_backup.fits[1]',
'POL0V_backup.fits[1]',
'P0lin_sci.fits,P60lin_sci.fits,P120lin_sci.fits',
fitgeometry='rscale',
calctype='double',
interpolant='linear',
fluxconserve=self.yes)
sregister('POL*_backup.fits[1]',
'POL0V_backup.fits[1]',
'P0drz_sci.fits,P60drz_sci.fits,P120drz_sci.fits',
fitgeometry='rscale',
calctype='double',
interpolant='drizzle',
fluxconserve=self.yes)
sregister('POL*_backup.fits[2]',
'POL0V_backup.fits[1]',
'P0lin_wht.fits,P60lin_wht.fits,P120lin_wht.fits',
fitgeometry='rscale',
calctype='double',
interpolant='linear',
fluxconserve=self.yes)
sregister('POL*_backup.fits[2]',
'POL0V_backup.fits[1]',
'P0drz_wht.fits,P60drz_wht.fits,P120drz_wht.fits',
fitgeometry='rscale',
calctype='double',
interpolant='drizzle',
fluxconserve=self.yes)
#average in 10x10x10 blocks
blkavg('P0lin_sci.fits',
'P0lin_smt.fits',
b1=10,
b2=10,
b3=10,
option='average')
blkavg('P60lin_sci.fits',
'P60lin_smt.fits',
b1=10,
b2=10,
b3=10,
option='average')
blkavg('P120lin_sci.fits',
'P120lin_smt.fits',
b1=10,
b2=10,
b3=10,
option='average')
blkavg('P0drz_sci.fits',
'P0drz_smt.fits',
b1=10,
b2=10,
b3=10,
option='average')
blkavg('P60drz_sci.fits',
'P60drz_smt.fits',
b1=10,
b2=10,
b3=10,
option='average')
blkavg('P120drz_sci.fits',
'P120drz_smt.fits',
b1=10,
b2=10,
b3=10,
option='average')
for ff in im:
f.write(ff + '\n')
f.close()
# -- set conv width based on filter name (IR = 2.5; optical=3.5)
fheader = pyfits.getheader(im[0])
instr = fheader['INSTRUME']
if instr == 'WFC3':
filtname = fheader['FILTER']
else: #assuming ACS
filtname = fheader['FILTER1']
if filtname[0] == 'C': filtname = fheader['FILTER2']
if filtname[0:2] == 'F1': conv_wid = 2.5
else: conv_wid = 3.5
# -- run tweakreg (SExtractor catalog: XWIN cols 8/9; XORG cols 2/3)
iraf.unlearn('tweakreg')
if USE_REF:
tweakreg.TweakReg('@imlist.dat',catfile=catfilename,xcol=xcol,ycol=ycol,refimage=irefim,refcat=irefcat,refxyunits='pixels',refxcol=rxcol,refycol=rycol, runfile=logfile, \
conv_width=conv_wid,searchrad=3.0,updatehdr=True,nclip=7,shiftfile=True,outshifts='shift.dat', see2dplot=False,residplot='No plot')
else:
tweakreg.TweakReg('@imlist.dat',catfile=catfilename,xcol=xcol,ycol=ycol,conv_width=conv_wid,searchrad=3.0, runfile=logfile, \
updatehdr=True,nclip=7,shiftfile=True,outshifts='shift.dat',see2dplot=False,residplot='No plot')
# -- remove unwanted tweakreg files
tmp = np.concatenate((glob.glob('*.coo'), glob.glob('*catalog.match'),
glob.glob('shift*.fits')))
for ff in tmp:
os.remove(ff)
# Set source detection Parameters
# Set the conv_width values for optical/ir images
vis_cw = 3.5
ir_cw = 2.5
# Set the thresholds for source detection for optical/ir images
vis_thresh = 5.
ir_thresh = 3.
if len(vis_ims)>0:
print 'Aligning optical images'
tweakreg.TweakReg(sorted(vis_ims), updatehdr=True, expand_refcat=False,
enforce_user_order=True,refimage=ref,
imagefindcfg={'threshold':vis_thresh.,'conv_width':vis_cw},
refimagefindcfg={'threshold':thresh,'conv_width':cw},
refcat=cat,shiftfile=True,outshifts='vis_shifts.txt',
wcsname=wcsname, interactive=False,see2dplot=False,
fitgeometry='general',reusename=True,separation=0.0)
if len(ir_ims)>0:
print 'Aligning IR images'
tweakreg.TweakReg(sorted(ir_ims), updatehdr=True, expand_refcat=False,
enforce_user_order=True,refimage=ref,
imagefindcfg={'threshold':ir_thresh,'conv_width':ir_cw},
refimagefindcfg={'threshold':thresh,'conv_width':cw},
refcat=cat,shiftfile=True,outshifts='ir_shifts.txt',
wcsname=wcsname, interactive=False, see2dplot=False,
fitgeometry='general',reusename=True,separation=0.0)
