def applyShiftsSA(visit):
sa_out = '%s_simplematch.out' % visit
drzs = {}
print('Reading %s...' % sa_out)
for i in [j.split() for j in open(sa_out).readlines()]:
drz = i[0].replace('_sa.cat', '.fits')
drzs[drz] = [float(i[1]), float(i[2]), float(i[3])]
print("Applying shifts...")
with open('sa_shifts.txt', 'a') as sas:
for d, s in drzs.items():
if os.path.exists(d):
if d[:6] == visit:
if s:
dx, dy, dt = s
wcs = HSTWCS(fits.open(d))
dxp = round(dx / wcs.pscale, 3)
dyp = round(dy / wcs.pscale, 3)
dtp = round(dt, 3)
offsets = [d, dxp, dyp, dtp]
sas.write('%s %.3f %.3f %.3f\n' % (d, dxp, dyp, dtp))
print(d, dxp, dyp, dtp)
updatehdr.updatewcs_with_shift(d,
d,
wcsname='DRZWCS',
xsh=dxp,
ysh=dyp,
rot=dtp,
scale=1.0,
force=True)
def shift_wcs(im):
try:
im_wcs = wcs.WCS(fits.getheader(im, 1))
except:
raise AssertionError('COULDN\'T FIND WCS FOR {}'.format(im))
if im_wcs.wcs.name == 'HSC':
print '{} already aligned to HSC, skipping'.format(im)
return
f = [ln.strip('\n') for ln in open('hsc_shifts.txt').readlines()][-1]
xsh, ysh, rot, scl, xrms, yrms = f.split()[1:]
ref = 'hsc_shifts_wcs.fits'
print im
try:
updatehdr.updatewcs_with_shift(im,
ref,
xsh=xsh,
ysh=ysh,
rot=rot,
scale=scl,
wcsname='HSC',
force=False,
sciext='SCI')
tweakback.tweakback(im,
force=False,
origwcs='DRZWCS',
newname='HSC',
wcsname='HSC')
except:
print 'COULDN\'T UPDATE {}, ERRORING'.format(im)
raise
def shift_wcs(im):
try:
im_wcs = wcs.WCS(fits.getheader(im,1))
except:
raise AssertionError('COULDN\'T FIND WCS FOR {}'.format(im))
if im_wcs.wcs.name == 'HSC':
print '{} already aligned to HSC, skipping'.format(im)
return
f = [ln.strip('\n') for ln in open('hsc_shifts.txt').readlines()][-1]
xsh, ysh, rot, scl, xrms, yrms = f.split()[1:]
ref = 'hsc_shifts_wcs.fits'
print im
try:
updatehdr.updatewcs_with_shift(im,ref,xsh=xsh,ysh=ysh,rot=rot,scale=scl,wcsname='HSC',force=False,sciext='SCI')
tweakback.tweakback(im,force=False,origwcs='DRZWCS',newname='HSC',wcsname='HSC')
except:
print 'COULDN\'T UPDATE {}, ERRORING'.format(im)
raise
def apply_shift(self,
database,
input_image="",
suffix='geo',
force_north_up=True):
# Apply the shifts to input image WCS header keywords
# Once the database has been calculated, one can use this to shift many
# other images.
# First read the shifts (in arcsec) and rotation (in degrees)
xshift, yshift, xrot = self.read_shifts(database)
# Then calculate the shifts in INPUT IMAGE PIXELS
xshift_pix = xshift / self.input_pixscale
yshift_pix = yshift / self.input_pixscale
print "xshift_pix, yshift_pix", xshift_pix, yshift_pix
# Now call updatehdr.updatewcs_with_shift
if not input_image:
input_image = self.input_image
output_image = os.path.splitext(input_image)[0] + '_%s.fits' % suffix
if os.path.exists(output_image):
print "Removing %s..." % output_image
os.remove(output_image)
os.system('cp %s %s' % (input_image, output_image))
updatehdr.updatewcs_with_shift(output_image,
input_image,
rot=xrot,
scale=1.0,
xsh=-xshift_pix,
ysh=-yshift_pix,
force=True,
verbose=True)
if force_north_up:
h = pyfits.open(output_image, mode='update')
pixscale = np.sqrt(h[0].header['cd1_1']**2 +
h[0].header['cd1_2']**2) # in degrees
h[0].header['cd1_1'] = np.sign(h[0].header['cd1_1']) * pixscale
h[0].header['cd1_2'] = 0.
h[0].header['cd2_1'] = 0.
h[0].header['cd2_2'] = np.sign(h[0].header['cd2_2']) * pixscale
h[0].header['orientat'] = 0.
h.flush()
h.close()
def refineShiftMCMC(drzfile):
"""
Refine shifts using MCMC
"""
dsn = drzfile[:9]
wcs = HSTWCS(fits.open(drzfile))
wcsn = fits.getval(drzfile, 'wcsname')
try:
refcat = np.loadtxt('%s_drz_sci_sa_ref_match.cat' % dsn,
usecols=(1, 2))
imgcat = np.loadtxt('%s_drz_sci_sa_match.cat' % dsn, usecols=(1, 2))
refcatw = wcs.all_world2pix(refcat, 1)
imgcatw = wcs.all_world2pix(imgcat, 1)
ox, oy = wcs.wcs.crpix.tolist()
offset, err = mcmcShifts.findOffsetMCMC(imgcatw,
refcatw,
maxShift=(10, 10, 0.3),
rotOrigin=(ox, oy),
precision=0.01,
visualize=False)
print(drzfile, offset, err)
dxp, dyp, dtp = offset
updatehdr.updatewcs_with_shift(drzfile,
drzfile,
wcsname='DRZWCS',
xsh=dxp,
ysh=dyp,
rot=dtp,
scale=1.0,
force=True)
return offset
except UserWarning:
pass
def get_align_to_subaru(sci='M0416_Ks_c1_mp_avg.fits',
wht='M0416_Ks_c1_mp_exp.fits',
field='',
clean=True,
toler=3,
verbose=False,
fitgeometry='shift',
shift_max=20,
rms_max=1.1,
rot_max=2,
rot_only=True,
THRESH=2,
align_data=None):
"""
Align HAWK-I images to the FF Subaru astrometric reference catalogs
"""
#sci='M0416_Ks_c1_mp_avg.fits'; wht='M0416_Ks_c1_mp_exp.fits'
### Make object catalog
se = threedhst.sex.SExtractor()
se.aXeParams()
se.copyConvFile()
se.overwrite = True
se.options['CHECKIMAGE_TYPE'] = 'NONE'
if wht is None:
se.options['WEIGHT_TYPE'] = 'NONE'
else:
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
se.options['WEIGHT_IMAGE'] = wht
se.options['FILTER'] = 'Y'
se.options['DETECT_THRESH'] = '%d' % (THRESH)
se.options['ANALYSIS_THRESH'] = '%d' % (THRESH)
se.options['MAG_ZEROPOINT'] = '26.0'
#### Run SExtractor on direct and alignment images
## direct image
se.options['CATALOG_NAME'] = 'direct.cat'
status = se.sextractImage(sci)
threedhst.sex.sexcatRegions('direct.cat', 'direct.reg', format=2)
directCat = threedhst.sex.mySexCat('direct.cat')
#### Get the X/Y coords of the reference catalog
#head = pyfits.getheader(sci, 0)
#wcs = pywcs.WCS(head)
if 'M0416' in sci:
ra_list, dec_list, mag = np.loadtxt(
os.getenv('HAWKI') +
'/FrontierFields/HST/hlsp_frontier_subaru_suprimecam_macs0416-astrom_R_v1_cat.txt',
unpack=True)
if ('c4' in sci):
ra_list, dec_list, mag = np.loadtxt(
os.getenv('HAWKI') +
'/FrontierFields/HST/M0416/macs0416_f814w_radec.cat',
unpack=True)
#
if 'M0717' in sci:
ra_list, dec_list, mag = np.loadtxt('subaru.radec', unpack=True)
if ('M1149' in sci) | (field == 'M1149'):
ra_list, dec_list, mag = np.loadtxt(
'/Users/brammer/Research/VLT/HAWKI/MACS1149/hlsp_frontier_subaru_suprimecam_macs1149-astrom_R_v1_cat.txt',
unpack=True)
if 'A2744' in sci:
ra_list, dec_list, mag = np.loadtxt(
os.getenv('HAWKI') +
'/FrontierFields/HST/hlsp_frontier_subaru_suprimecam_abell2744-astrom_i_v1_cat.txt',
unpack=True)
if ('c1' in sci) | ('c4' in sci):
ra_list, dec_list, mag = np.loadtxt(
os.getenv('HAWKI') +
'/FrontierFields/HST/abell2744_f814w_radec.cat',
unpack=True)
if align_data is not None:
ra_list, dec_list, mag = align_data
im = pyfits.open(sci)
print sci
sh = im[0].shape
head = im[0].header
head['CUNIT1'] = 'deg'
head['CUNIT2'] = 'deg'
wcs = pywcs.WCS(head)
x_image, y_image = wcs.wcs_sky2pix(ra_list, dec_list, 1)
try:
x_image, y_image = wcs.wcs_sky2pix(ra_list, dec_list, 1)
except:
x_image, y_image = wcs.wcs_world2pix(ra_list, dec_list, 1)
ok = (x_image > 0) & (y_image > 0) & (x_image < sh[1]) & (y_image < sh[1])
x_image, y_image = x_image[ok], y_image[ok]
fpr = open('align.reg', 'w')
fpr.write('image\n')
for i in range(ok.sum()):
fpr.write('circle(%.6f, %.6f,0.3") # color=magenta\n' %
(x_image[i], y_image[i]))
fpr.close()
# x_image, y_image = [], []
#
# for ra, dec in zip(ra_list, dec_list):
# x, y = wcs.wcs_sky2pix([[ra, dec]], 1)[0]
# if (x > 0) & (y > 0) & (x < sh[1]) & (y < sh[1]):
# x_image.append(x)
# y_image.append(y)
alignCat = catIO.EmptyCat()
alignCat['X_IMAGE'] = np.array(x_image)
alignCat['Y_IMAGE'] = np.array(y_image)
xshift = 0
yshift = 0
rot = 0
scale = 1.
xrms = 2
yrms = 2
NITER = 5
IT = 0
while (IT < NITER):
IT = IT + 1
#### Get x,y coordinates of detected objects
## direct image
fp = open('direct.xy', 'w')
for i in range(len(directCat.X_IMAGE)):
fp.write('%s %s\n' % (directCat.X_IMAGE[i], directCat.Y_IMAGE[i]))
fp.close()
## alignment image
fp = open('align.xy', 'w')
for i in range(len(alignCat.X_IMAGE)):
fp.write('%s %s\n' % (np.float(alignCat.X_IMAGE[i]) + xshift,
np.float(alignCat.Y_IMAGE[i]) + yshift))
fp.close()
iraf.flpr()
iraf.flpr()
iraf.flpr()
#### iraf.xyxymatch to find matches between the two catalogs
pow = toler * 1.
try:
os.remove('align.match')
except:
pass
status1 = iraf.xyxymatch(input="direct.xy",
reference="align.xy",
output="align.match",
tolerance=2**pow,
separation=0,
verbose=iraf.yes,
Stdout=1)
nmatch = 0
while status1[-1].startswith('0') | (nmatch < 10) | (float(
status1[-3].split()[1]) > 40):
pow += 1
os.remove('align.match')
status1 = iraf.xyxymatch(input="direct.xy",
reference="align.xy",
output="align.match",
tolerance=2**pow,
separation=0,
verbose=iraf.yes,
Stdout=1)
#
nmatch = 0
for line in open('align.match').xreadlines():
nmatch += 1
if verbose:
for line in status1:
print line
#### Compute shifts with iraf.geomap
iraf.flpr()
iraf.flpr()
iraf.flpr()
try:
os.remove("align.map")
except:
pass
status2 = iraf.geomap(input="align.match",
database="align.map",
fitgeometry=fitgeometry,
interactive=iraf.no,
xmin=iraf.INDEF,
xmax=iraf.INDEF,
ymin=iraf.INDEF,
ymax=iraf.INDEF,
maxiter=10,
reject=2.0,
Stdout=1)
if verbose:
for line in status2:
print line
#fp = open(root+'.iraf.log','a')
#fp.writelines(status1)
#fp.writelines(status2)
#fp.close()
#### Parse geomap.output
fp = open("align.map", "r")
for line in fp.readlines():
spl = line.split()
if spl[0].startswith('xshift'):
xshift += float(spl[1])
if spl[0].startswith('yshift'):
yshift += float(spl[1])
if spl[0].startswith('xrotation'):
rot = float(spl[1])
if spl[0].startswith('xmag'):
scale = float(spl[1])
if spl[0].startswith('xrms'):
xrms = float(spl[1])
if spl[0].startswith('yrms'):
yrms = float(spl[1])
fp.close()
#os.system('wc align.match')
print 'Shift iteration #%d, xshift=%f, yshift=%f, rot=%f, scl=%f (rms: %5.2f,%5.2f)' % (
IT, xshift, yshift, rot, scale, xrms, yrms)
os.system(
'cat align.match | grep -v "\#" | grep [0-9] | awk \'{print "circle(", $1, ",", $2, ",4) # color=green"}\' > d.reg'
)
os.system(
'cat align.match | grep -v "\#" | grep [0-9] | awk \'{print "circle(", $3, ",", $4, ",4) # color=magenta"}\' > a.reg'
)
shutil.copy('align.map', sci.replace('.fits', '.align.map'))
shutil.copy('align.match', sci.replace('.fits', '.align.match'))
#### Cleanup
if clean:
rmfiles = [
'align.cat', 'align.map', 'align.match', 'align.reg', 'align.xy',
'direct.cat', 'direct.reg', 'direct.xy'
]
for file in rmfiles:
try:
os.remove(file)
except:
pass
fp = open(sci.replace('.fits', '.align.info'), 'w')
fp.write('# image xshift yshift rot scale xrms yrms\n')
fp.write('%s %.3f %.3f %.4f %.4f %.3f %.3f\n' %
(sci, xshift, yshift, rot, scale, xrms, yrms))
if (np.abs(xshift) > shift_max) | (np.abs(yshift) > shift_max) | (
xrms > rms_max) | (yrms > rms_max):
print 'Shifts out of allowed range. Run again with increased shift_max to accept.'
#return xshift, yshift, rot, scale, xrms, yrms
## Add a small shift that should come out easily with another
## shift iteration
xshift, yshift, rot, scale, xrms, yrms = 2, 2, 0, 1.0, -99, -99
for file in [sci, wht]:
if ('r' in fitgeometry) & rot_only:
xshift, yshift = 0, 0
#apply_offsets(file, [[xshift, yshift, rot, scale]])
from drizzlepac import updatehdr
updatehdr.updatewcs_with_shift(file,
sci,
wcsname='DRZWCS',
rot=rot,
scale=scale,
xsh=xshift,
ysh=yshift,
fit=None,
xrms=xrms,
yrms=yrms,
verbose=False,
force=True,
sciext=0)
if '_dr' in sci:
im = pyfits.open(sci)
h = im[0].header
for i in range(h['NDRIZIM']):
flt_str = h['D%03dDATA' % (i + 1)]
if 'sci,2' in flt_str:
continue
#
flt_im = flt_str.split('[')[0]
ext = int(flt_str.split('[')[1][:-1].split(',')[1])
updatehdr.updatewcs_with_shift(flt_im,
sci,
wcsname='GTWEAK',
rot=rot,
scale=scale,
xsh=xshift,
ysh=yshift,
fit=None,
xrms=xrms,
yrms=yrms,
verbose=False,
force=True,
sciext='SCI')
# im = pyfits.open(file, mode='update')
# wcs = pywcs.WCS(im[0].header)
# wcs.rotateCD(-rot)
# wcs.wcs.cd /= scale
# #
# im[0].header['CRPIX1'] += xshift
# im[0].header['CRPIX2'] += yshift
# #
# for i in [0,1]:
# for j in [0,1]:
# im[0].header['CD%d_%d' %(i+1, j+1)] = wcs.wcs.cd[i,j]
# #
# im.flush()
return xshift, yshift, rot, scale, xrms, yrms
def auto_run(root='j023507-040202', flag_global_crs=False):
import os
import glob
import numpy as np
import astropy.io.fits as pyfits
import astropy.wcs as pywcs
from drizzlepac import updatehdr
from stwcs import updatewcs
from grizli import utils, prep
from grizli.pipeline import auto_script
utils.set_warnings()
visit_file = '{0}_visits.npy'.format(root)
visits, all_groups, info = np.load(visit_file)
# Something wrong with some files with bad shifts, reset wcs
for visit in visits:
for file in visit['files']:
utils.fetch_hst_calibs(
file, calib_types=['IDCTAB', 'NPOLFILE', 'IMPHTTAB'])
updatewcs.updatewcs(file, verbose=True, use_db=False)
# Apply shifts
shift_log = '{0}_shifts.log'.format(visit['product'])
if os.path.exists(shift_log):
sh = utils.read_catalog(shift_log)
flt0 = pyfits.open(sh['flt'][0])
wcs_ref = pywcs.WCS(flt0['SCI', 1].header, fobj=flt0, relax=True)
shift_dict = {}
for i in range(len(sh)):
shift_dict[sh['flt'][i]] = [sh['xshift'][i], sh['yshift'][i]]
prep.apply_tweak_shifts(wcs_ref,
shift_dict,
grism_matches={},
verbose=False)
# Redrizzle mosaics
prep.drizzle_overlaps(visits,
check_overlaps=False,
skysub=False,
static=False,
pixfrac=0.5,
scale=None,
final_wcs=False,
fetch_flats=False,
final_rot=None,
include_saturated=True)
####### Alignment
os.system('rm *wcs.*')
# Radec
master_radec = '{0}/../../{1}_master.radec'.format(os.getcwd(), root)
if not os.path.exists(master_radec):
master_radec = None
ref_catalog = 'USER'
if root.startswith('cos-'):
hsc = '{0}/../../{1}'.format(os.getcwd(),
'hsc-udeep-i25_corr_cosmos.radec')
if os.path.exists(hsc):
master_radec = hsc
ref_catalog = 'HSC'
elif root.startswith('uds-'):
hsc = '{0}/../../{1}'.format(os.getcwd(),
'hsc-udeep-sxds_corr_uds.radec')
if os.path.exists(hsc):
master_radec = hsc
ref_catalog = 'HSC'
parent_radec = '{0}/../../{1}_parent.radec'.format(os.getcwd(), root)
if not os.path.exists(parent_radec):
parent_radec = None
if master_radec is not None:
radec = master_radec
elif parent_radec is not None:
radec = parent_radec
else:
radec = None
if radec is None:
needs_gaia = True
else:
needs_gaia = False
REFERENCE = 'GAIA'
REFERENCE = 'PS1'
print('master RADEC file: ', radec)
thresh = 2.5
for visit in visits:
# Clean catalogs
files = glob.glob('{0}.*'.format(visit['product']))
for file in files:
os.remove(file)
# Generate GAIA alignment catalog at the observation epoch
clip = 120
clip = -1
if needs_gaia:
flt = pyfits.open(visit['files'][0])
h = flt['SCI', 1].header
ra_i, dec_i = h['CRVAL1'], h['CRVAL2']
radec, ref_catalog = prep.get_radec_catalog(
ra=ra_i,
dec=dec_i,
product=visit['product'],
date=flt[0].header['EXPSTART'],
date_format='mjd',
reference_catalogs=[REFERENCE],
radius=5.)
flt.close()
if REFERENCE == 'GAIA':
mag_limits = [16, 20]
else:
mag_limits = [18, 22]
#clip = 50
if '_flc' in visit['files'][0]:
triangle_size_limit = [5, 4000 * np.sqrt(2)]
else:
triangle_size_limit = [5, 1300]
else:
mag_limits = [19, 23]
triangle_size_limit = [5, 1300]
# Remake catalogs
cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh)
# Redo alignment
try:
print('XXX clip', clip, mag_limits, triangle_size_limit)
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=mag_limits,
simple=False,
max_err_percentile=80,
clip=clip,
outlier_threshold=5,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
except:
print('First align failed! Relax parameters')
try:
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=[10, 20],
simple=False,
max_err_percentile=99,
clip=160,
outlier_threshold=20,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
except:
try:
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=[10, 20],
simple=False,
max_err_percentile=99,
clip=160,
outlier_threshold=40,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
except:
radec = '{0}_ps1.radec'.format(visit['product'])
ref_catalog = 'PS1'
result = prep.align_drizzled_image(
root=visit['product'],
radec=radec,
mag_limits=mag_limits,
simple=False,
max_err_percentile=80,
clip=120,
outlier_threshold=5,
rms_limit=2.5,
triangle_size_limit=triangle_size_limit)
#continue
orig_wcs, drz_wcs, out_shift, out_rot, out_scale = result
# Propagate shifts
for file in visit['files']:
updatehdr.updatewcs_with_shift(file,
str('{0}_wcs.fits'.format(
visit['product'])),
xsh=out_shift[0],
ysh=out_shift[1],
rot=out_rot,
scale=out_scale,
wcsname=ref_catalog,
force=True,
reusename=True,
verbose=True,
sciext='SCI')
### Bug in astrodrizzle? Dies if the FLT files don't have MJD-OBS
### keywords
im = pyfits.open(file, mode='update')
im[0].header['MJD-OBS'] = im[0].header['EXPSTART']
im.flush()
# Redrizzle mosaics again including new shifts
prep.drizzle_overlaps(visits,
check_overlaps=False,
skysub=False,
static=False,
pixfrac=0.8,
scale=None,
final_wcs=False,
fetch_flats=False,
final_rot=None)
# Remake catalogs
thresh = 2.5
for visit in visits:
# Remake catalogs
cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh)
prep.table_to_regions(cat, '{0}.cat.reg'.format(visit['product']))
prep.table_to_radec(cat, '{0}.cat.radec'.format(visit['product']))
# Update visits file
v = auto_script.get_visit_exposure_footprints(visit_file=visit_file,
check_paths=['./', '../RAW'],
simplify=1.e-6)
if flag_global_crs:
# Assume everything at same orient
pass
if False:
# Mosaic
auto_script.drizzle_overlaps(root,
filters=['F160W'],
min_nexp=1,
pixfrac=0.8,
scale=0.1,
make_combined=False,
ref_image=None,
static=False)
def SingleStarReg(imfile,
ra,
dec,
wcsname='SINGLESTAR',
computesig=False,
refim=None,
threshmin=0.5,
peakmin=None,
peakmax=None,
searchrad=5.0,
nsigma=1.5,
fluxmin=None,
fluxmax=None,
verbose=True,
clobber=True):
""" Update the WCS of imfile so that it matches the WCS of the refimfile,
using a single star to define the necessary shift.
If xy or xyref are provided, then assume the star is located within
searchrad arcsec of that position in the imfile or the refimfile,
respectively. If either of those pixel coordinates are not provided, then
use the brightest star in the image.
"""
# noinspection PyUnresolvedReferences
from numpy import deg2rad, sqrt, cos, where, median
from astropy.io import ascii
from drizzlepac.updatehdr import updatewcs_with_shift
from numpy import unique
if refim is None: refim = imfile
if computesig == False:
skysigma = getskysigma(imfile)
if verbose:
print(("sndrizipipe.register.SingleStarReg: "
" Manually computed sky sigma for %s as %.5e" %
(imfile, skysigma)))
else:
skysigma = 0.0
# locate stars in imfile, pick out the brightest one
topdir = os.path.abspath('.')
imfiledir = os.path.dirname(os.path.abspath(imfile))
imfilebase = os.path.basename(imfile)
os.chdir(imfiledir)
# Iterate the source-finding algorithm with progressively smaller threshold
# values. This helps to ensure that we correctly locate the single bright
# source in the image
xycatfile = None
threshold = 200.
while threshold >= threshmin:
try:
xycatfile = mkSourceCatalog(imfilebase,
computesig=computesig,
skysigma=skysigma,
nsigma=nsigma,
threshold=threshold,
peakmin=peakmin,
peakmax=peakmax,
fluxmin=fluxmin,
fluxmax=fluxmax)[0]
# The source finder succeeded!
radeccatfile = xycatfile.replace('xy.coo', 'radec.coo')
break
except NameError:
# the source finder failed, try again with a lower threshold
threshold /= 2.
continue
if xycatfile is None:
print(( "Failed to generate a clean source catalog for %s"%imfile + \
" using threshmin = %.3f"%threshmin ))
import pdb
pdb.set_trace()
raise RuntimeError(
"Failed to generate a clean source catalog for %s"%imfile + \
" using threshmin = %.3f"%threshmin )
xycat = ascii.read(xycatfile)
radeccat = ascii.read(radeccatfile)
if verbose:
print(("Located %i sources with threshold=%.1f sigma" %
(len(xycat), threshold)))
os.chdir(topdir)
# compute the approximate separation in arcsec from the target ra,dec
# to each of the detected sources, then limit to those within searchrad
rasrc, decsrc = radeccat['col1'], radeccat['col2']
darcsec = sqrt(((rasrc - ra) * cos(deg2rad(dec)))**2 +
(decsrc - dec)**2) * 3600.
inear = where(darcsec <= searchrad)[0]
if verbose:
print((" %i of these sources are within %.1f arcsec of the target" %
(len(inear), searchrad)))
# identify the brightest source within searchrad arcsec of the target
ibrightest = inear[xycat['col3'][inear].argmax()]
xfnd, yfnd = [xycat['col1'][ibrightest], xycat['col2'][ibrightest]]
if verbose:
brightratio = xycat['col3'][ibrightest] / median(xycat['col3'][inear])
print((
" The brightest of these sources is %.1fx brighter than the median."
% brightratio))
# The TweakReg imagefind algorithm sometimes misses the true center
# catastrophically. Here we use a centroiding algorithm to re-center the
# position on the star, checking for a large offset.
# Note that we are using numpy arrays, so the origin is (0,0) and we need
# to correct the cntrd output to the (1,1) origin used by pyfits and drizzlepac.
imdat = pyfits.getdata(imfile)
fwhmpix = getfwhmpix(imfile)
xcntrd, ycntrd = cntrd(imdat, xfnd, yfnd, fwhmpix)
if xcntrd == -1:
if verbose: print('Recentering within a 5-pixel box')
xcntrd, ycntrd = cntrd(imdat, xfnd, yfnd, fwhmpix, extendbox=5)
assert ((xcntrd > 0) &
(ycntrd > 0)), "Centroid recentering failed for %s" % imfile
xcntrd += 1
ycntrd += 1
dxcntrd = xfnd - xcntrd
dycntrd = yfnd - ycntrd
if verbose > 9:
# plot the centroid shift and save a .png image
from matplotlib import pylab as pl
from matplotlib import cm
pl.clf()
vmin = imdat[ycntrd - 10:ycntrd + 10, xcntrd - 10:xcntrd + 10].min()
vmax = imdat[ycntrd - 10:ycntrd + 10, xcntrd - 10:xcntrd + 10].max()
pl.imshow(imdat, cmap=cm.Greys, vmin=vmin, vmax=vmax)
pl.plot(xfnd - 1,
yfnd - 1,
'r+',
ms=12,
mew=1.5,
label='drizzlepac.ndfind source position: %.3f, %.3f' %
(xfnd, yfnd))
pl.plot(xcntrd - 1,
ycntrd - 1,
'gx',
ms=12,
mew=1.5,
label='register.cntrd recentered position: %.3f, %.3f' %
(xcntrd, ycntrd))
pl.title("Centroiding shift : dx = %.3f dy = %.3f" %
(dxcntrd, dycntrd))
ax = pl.gca()
ax.set_xlim(xcntrd - 10, xcntrd + 10)
ax.set_ylim(ycntrd - 10, ycntrd + 10)
pl.colorbar()
ax.set_xlabel('X (pixels)')
ax.set_ylabel('Y (pixels)')
ax.legend(loc='upper right', numpoints=1, frameon=False)
pl.draw()
outpng = imfile.replace('.fits', '_recenter.png')
pl.savefig(outpng)
print(("Saved a recentering image as " + outpng))
# locate the appropriate extensions for updating
hdulist = pyfits.open(imfile)
sciextlist = [hdu.name for hdu in hdulist if 'WCSAXES' in hdu.header]
# convert the target position from ra,dec to x,y
if len(sciextlist) > 0:
imwcs = stwcs.wcsutil.HSTWCS(hdulist, ext=(sciextlist[0], 1))
else:
sciextlist = ['PRIMARY']
imwcs = stwcs.wcsutil.HSTWCS(hdulist, ext=None)
xref, yref = imwcs.wcs_world2pix(ra, dec, 1)
# If the new centroid position differs substantially from the original
# ndfind position, then update the found source position
# (i.e. in cases of catastrophic ndfind failure)
if (abs(dxcntrd) > 0.5) or (abs(dycntrd) > 0.5):
xfnd = xcntrd
yfnd = ycntrd
# compute the pixel shift from the xy position found in the image
# to the reference (target) xy position
xshift = xfnd - xref
yshift = yfnd - yref
# apply that shift to the image wcs
for sciext in unique(sciextlist):
print(("Updating %s ext %s with xshift,yshift = %.5f %.5f" %
(imfile, sciext, xshift, yshift)))
updatewcs_with_shift(imfile,
refim,
wcsname=wcsname,
rot=0.0,
scale=1.0,
xsh=xshift,
ysh=yshift,
fit=None,
xrms=None,
yrms=None,
verbose=verbose,
force=clobber,
sciext=sciext)
hdulist.close()
return (wcsname)
def SingleStarReg( imfile, ra, dec, wcsname='SINGLESTAR',
computesig=False, refim=None,
threshmin=0.5, peakmin=None, peakmax=None, searchrad=5.0,
nsigma=1.5, fluxmin=None, fluxmax=None,
verbose=True, clobber=True ):
""" Update the WCS of imfile so that it matches the WCS of the refimfile,
using a single star to define the necessary shift.
If xy or xyref are provided, then assume the star is located within
searchrad arcsec of that position in the imfile or the refimfile,
respectively. If either of those pixel coordinates are not provided, then
use the brightest star in the image.
"""
# noinspection PyUnresolvedReferences
from numpy import deg2rad, sqrt, cos, where, median
from astropy.io import ascii
from drizzlepac.updatehdr import updatewcs_with_shift
from numpy import unique
if refim is None : refim = imfile
if computesig==False :
skysigma = getskysigma( imfile )
if verbose : print(("sndrizipipe.register.SingleStarReg: "
" Manually computed sky sigma for %s as %.5e"%(imfile,skysigma)))
else :
skysigma=0.0
# locate stars in imfile, pick out the brightest one
topdir = os.path.abspath( '.' )
imfiledir = os.path.dirname( os.path.abspath(imfile) )
imfilebase = os.path.basename( imfile )
os.chdir( imfiledir )
# Iterate the source-finding algorithm with progressively smaller threshold
# values. This helps to ensure that we correctly locate the single bright
# source in the image
xycatfile = None
threshold = 200.
while threshold >= threshmin :
try :
xycatfile = mkSourceCatalog(
imfilebase, computesig=computesig, skysigma=skysigma, nsigma=nsigma,
threshold=threshold, peakmin=peakmin, peakmax=peakmax,
fluxmin=fluxmin, fluxmax= fluxmax )[0]
# The source finder succeeded!
radeccatfile = xycatfile.replace('xy.coo','radec.coo')
break
except NameError :
# the source finder failed, try again with a lower threshold
threshold /= 2.
continue
if xycatfile is None :
print(( "Failed to generate a clean source catalog for %s"%imfile + \
" using threshmin = %.3f"%threshmin ))
import pdb; pdb.set_trace()
raise RuntimeError(
"Failed to generate a clean source catalog for %s"%imfile + \
" using threshmin = %.3f"%threshmin )
xycat = ascii.read( xycatfile )
radeccat = ascii.read( radeccatfile )
if verbose :
print(("Located %i sources with threshold=%.1f sigma"%(len(xycat),threshold)))
os.chdir( topdir )
# compute the approximate separation in arcsec from the target ra,dec
# to each of the detected sources, then limit to those within searchrad
rasrc, decsrc = radeccat['col1'], radeccat['col2']
darcsec = sqrt( ((rasrc-ra)*cos(deg2rad(dec)))**2 + (decsrc-dec)**2 ) * 3600.
inear = where( darcsec <= searchrad )[0]
if verbose :
print((" %i of these sources are within %.1f arcsec of the target"%(len(inear),searchrad)))
# identify the brightest source within searchrad arcsec of the target
ibrightest = inear[ xycat['col3'][inear].argmax() ]
xfnd,yfnd = [ xycat['col1'][ibrightest], xycat['col2'][ibrightest] ]
if verbose :
brightratio = xycat['col3'][ibrightest] / median(xycat['col3'][inear])
print((" The brightest of these sources is %.1fx brighter than the median."%brightratio))
# The TweakReg imagefind algorithm sometimes misses the true center
# catastrophically. Here we use a centroiding algorithm to re-center the
# position on the star, checking for a large offset.
# Note that we are using numpy arrays, so the origin is (0,0) and we need
# to correct the cntrd output to the (1,1) origin used by pyfits and drizzlepac.
imdat = pyfits.getdata( imfile )
fwhmpix = getfwhmpix( imfile )
xcntrd, ycntrd = cntrd( imdat, xfnd, yfnd, fwhmpix )
if xcntrd==-1 :
if verbose : print('Recentering within a 5-pixel box')
xcntrd, ycntrd = cntrd( imdat, xfnd, yfnd, fwhmpix, extendbox=5 )
assert ((xcntrd>0) & (ycntrd>0)), "Centroid recentering failed for %s"%imfile
xcntrd += 1
ycntrd += 1
dxcntrd = xfnd-xcntrd
dycntrd = yfnd-ycntrd
if verbose >9 :
# plot the centroid shift and save a .png image
from matplotlib import pylab as pl
from matplotlib import cm
pl.clf()
vmin = imdat[ycntrd-10:ycntrd+10,xcntrd-10:xcntrd+10].min()
vmax = imdat[ycntrd-10:ycntrd+10,xcntrd-10:xcntrd+10].max()
pl.imshow( imdat, cmap=cm.Greys, vmin=vmin, vmax=vmax)
pl.plot( xfnd-1, yfnd-1, 'r+', ms=12, mew=1.5,
label='drizzlepac.ndfind source position: %.3f, %.3f'%(xfnd,yfnd) )
pl.plot( xcntrd-1, ycntrd-1, 'gx', ms=12, mew=1.5,
label='register.cntrd recentered position: %.3f, %.3f'%(xcntrd,ycntrd) )
pl.title( "Centroiding shift : dx = %.3f dy = %.3f"%(dxcntrd,dycntrd) )
ax = pl.gca()
ax.set_xlim( xcntrd-10, xcntrd+10 )
ax.set_ylim( ycntrd-10, ycntrd+10 )
pl.colorbar()
ax.set_xlabel('X (pixels)')
ax.set_ylabel('Y (pixels)')
ax.legend( loc='upper right', numpoints=1, frameon=False )
pl.draw()
outpng = imfile.replace('.fits','_recenter.png')
pl.savefig(outpng)
print(("Saved a recentering image as " + outpng ))
# locate the appropriate extensions for updating
hdulist = pyfits.open( imfile )
sciextlist = [ hdu.name for hdu in hdulist if 'WCSAXES' in hdu.header ]
# convert the target position from ra,dec to x,y
if len(sciextlist)>0:
imwcs = stwcs.wcsutil.HSTWCS( hdulist, ext=(sciextlist[0],1) )
else:
sciextlist = ['PRIMARY']
imwcs = stwcs.wcsutil.HSTWCS( hdulist, ext=None )
xref, yref = imwcs.wcs_world2pix( ra, dec , 1)
# If the new centroid position differs substantially from the original
# ndfind position, then update the found source position
# (i.e. in cases of catastrophic ndfind failure)
if (abs(dxcntrd)>0.5) or (abs(dycntrd)>0.5) :
xfnd = xcntrd
yfnd = ycntrd
# compute the pixel shift from the xy position found in the image
# to the reference (target) xy position
xshift = xfnd - xref
yshift = yfnd - yref
# apply that shift to the image wcs
for sciext in unique(sciextlist):
print(("Updating %s ext %s with xshift,yshift = %.5f %.5f"%(
imfile,sciext, xshift, yshift )))
updatewcs_with_shift(imfile, refim, wcsname=wcsname,
rot=0.0, scale=1.0, xsh=xshift, ysh=yshift,
fit=None, xrms=None, yrms=None, verbose=verbose,
force=clobber, sciext=sciext )
hdulist.close()
return( wcsname )
def get_align_to_subaru(sci='M0416_Ks_c1_mp_avg.fits', wht='M0416_Ks_c1_mp_exp.fits', field='', clean=True, toler=3, verbose=False, fitgeometry='shift', shift_max=20, rms_max=1.1, rot_max=2, rot_only=True, THRESH=2, align_data=None):
"""
Align HAWK-I images to the FF Subaru astrometric reference catalogs
"""
#sci='M0416_Ks_c1_mp_avg.fits'; wht='M0416_Ks_c1_mp_exp.fits'
### Make object catalog
se = threedhst.sex.SExtractor()
se.aXeParams()
se.copyConvFile()
se.overwrite = True
se.options['CHECKIMAGE_TYPE'] = 'NONE'
if wht is None:
se.options['WEIGHT_TYPE'] = 'NONE'
else:
se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT'
se.options['WEIGHT_IMAGE'] = wht
se.options['FILTER'] = 'Y'
se.options['DETECT_THRESH'] = '%d' %(THRESH)
se.options['ANALYSIS_THRESH'] = '%d' %(THRESH)
se.options['MAG_ZEROPOINT'] = '26.0'
#### Run SExtractor on direct and alignment images
## direct image
se.options['CATALOG_NAME'] = 'direct.cat'
status = se.sextractImage(sci)
threedhst.sex.sexcatRegions('direct.cat', 'direct.reg', format=2)
directCat = threedhst.sex.mySexCat('direct.cat')
#### Get the X/Y coords of the reference catalog
#head = pyfits.getheader(sci, 0)
#wcs = pywcs.WCS(head)
if 'M0416' in sci:
ra_list, dec_list, mag = np.loadtxt(os.getenv('HAWKI')+'/FrontierFields/HST/hlsp_frontier_subaru_suprimecam_macs0416-astrom_R_v1_cat.txt', unpack=True)
if ('c4' in sci):
ra_list, dec_list, mag = np.loadtxt(os.getenv('HAWKI')+'/FrontierFields/HST/M0416/macs0416_f814w_radec.cat', unpack=True)
#
if 'M0717' in sci:
ra_list, dec_list, mag = np.loadtxt('subaru.radec', unpack=True)
if ('M1149' in sci) | (field == 'M1149'):
ra_list, dec_list, mag = np.loadtxt('/Users/brammer/Research/VLT/HAWKI/MACS1149/hlsp_frontier_subaru_suprimecam_macs1149-astrom_R_v1_cat.txt', unpack=True)
if 'A2744' in sci:
ra_list, dec_list, mag = np.loadtxt(os.getenv('HAWKI')+'/FrontierFields/HST/hlsp_frontier_subaru_suprimecam_abell2744-astrom_i_v1_cat.txt', unpack=True)
if ('c1' in sci) | ('c4' in sci):
ra_list, dec_list, mag = np.loadtxt(os.getenv('HAWKI')+'/FrontierFields/HST/abell2744_f814w_radec.cat', unpack=True)
if align_data is not None:
ra_list, dec_list, mag = align_data
im = pyfits.open(sci)
print sci
sh = im[0].shape
head = im[0].header
head['CUNIT1'] = 'deg'; head['CUNIT2'] = 'deg'
wcs = pywcs.WCS(head)
x_image, y_image = wcs.wcs_sky2pix(ra_list, dec_list, 1)
try:
x_image, y_image = wcs.wcs_sky2pix(ra_list, dec_list, 1)
except:
x_image, y_image = wcs.wcs_world2pix(ra_list, dec_list, 1)
ok = (x_image > 0) & (y_image > 0) & (x_image < sh[1]) & (y_image < sh[1])
x_image, y_image = x_image[ok], y_image[ok]
fpr = open('align.reg','w')
fpr.write('image\n')
for i in range(ok.sum()): fpr.write('circle(%.6f, %.6f,0.3") # color=magenta\n' %(x_image[i], y_image[i]))
fpr.close()
# x_image, y_image = [], []
#
# for ra, dec in zip(ra_list, dec_list):
# x, y = wcs.wcs_sky2pix([[ra, dec]], 1)[0]
# if (x > 0) & (y > 0) & (x < sh[1]) & (y < sh[1]):
# x_image.append(x)
# y_image.append(y)
alignCat = catIO.EmptyCat()
alignCat['X_IMAGE'] = np.array(x_image)
alignCat['Y_IMAGE'] = np.array(y_image)
xshift = 0
yshift = 0
rot = 0
scale = 1.
xrms = 2
yrms = 2
NITER = 5
IT = 0
while (IT < NITER):
IT = IT+1
#### Get x,y coordinates of detected objects
## direct image
fp = open('direct.xy','w')
for i in range(len(directCat.X_IMAGE)):
fp.write('%s %s\n' %(directCat.X_IMAGE[i],directCat.Y_IMAGE[i]))
fp.close()
## alignment image
fp = open('align.xy','w')
for i in range(len(alignCat.X_IMAGE)):
fp.write('%s %s\n' %(np.float(alignCat.X_IMAGE[i])+xshift,
np.float(alignCat.Y_IMAGE[i])+yshift))
fp.close()
iraf.flpr()
iraf.flpr()
iraf.flpr()
#### iraf.xyxymatch to find matches between the two catalogs
pow = toler*1.
try:
os.remove('align.match')
except:
pass
status1 = iraf.xyxymatch(input="direct.xy", reference="align.xy",
output="align.match",
tolerance=2**pow, separation=0, verbose=iraf.yes, Stdout=1)
nmatch = 0
while status1[-1].startswith('0') | (nmatch < 10) | (float(status1[-3].split()[1]) > 40):
pow+=1
os.remove('align.match')
status1 = iraf.xyxymatch(input="direct.xy", reference="align.xy",
output="align.match",
tolerance=2**pow, separation=0, verbose=iraf.yes, Stdout=1)
#
nmatch = 0
for line in open('align.match').xreadlines( ): nmatch += 1
if verbose:
for line in status1:
print line
#### Compute shifts with iraf.geomap
iraf.flpr()
iraf.flpr()
iraf.flpr()
try:
os.remove("align.map")
except:
pass
status2 = iraf.geomap(input="align.match", database="align.map",
fitgeometry=fitgeometry, interactive=iraf.no,
xmin=iraf.INDEF, xmax=iraf.INDEF, ymin=iraf.INDEF, ymax=iraf.INDEF,
maxiter = 10, reject = 2.0, Stdout=1)
if verbose:
for line in status2:
print line
#fp = open(root+'.iraf.log','a')
#fp.writelines(status1)
#fp.writelines(status2)
#fp.close()
#### Parse geomap.output
fp = open("align.map","r")
for line in fp.readlines():
spl = line.split()
if spl[0].startswith('xshift'):
xshift += float(spl[1])
if spl[0].startswith('yshift'):
yshift += float(spl[1])
if spl[0].startswith('xrotation'):
rot = float(spl[1])
if spl[0].startswith('xmag'):
scale = float(spl[1])
if spl[0].startswith('xrms'):
xrms = float(spl[1])
if spl[0].startswith('yrms'):
yrms = float(spl[1])
fp.close()
#os.system('wc align.match')
print 'Shift iteration #%d, xshift=%f, yshift=%f, rot=%f, scl=%f (rms: %5.2f,%5.2f)' %(IT, xshift, yshift, rot, scale, xrms, yrms)
os.system('cat align.match | grep -v "\#" | grep [0-9] | awk \'{print "circle(", $1, ",", $2, ",4) # color=green"}\' > d.reg')
os.system('cat align.match | grep -v "\#" | grep [0-9] | awk \'{print "circle(", $3, ",", $4, ",4) # color=magenta"}\' > a.reg')
shutil.copy('align.map', sci.replace('.fits', '.align.map'))
shutil.copy('align.match', sci.replace('.fits', '.align.match'))
#### Cleanup
if clean:
rmfiles = ['align.cat', 'align.map','align.match','align.reg','align.xy', 'direct.cat','direct.reg','direct.xy']
for file in rmfiles:
try:
os.remove(file)
except:
pass
fp = open(sci.replace('.fits', '.align.info'), 'w')
fp.write('# image xshift yshift rot scale xrms yrms\n')
fp.write('%s %.3f %.3f %.4f %.4f %.3f %.3f\n' %(sci, xshift, yshift, rot, scale, xrms, yrms))
if (np.abs(xshift) > shift_max) | (np.abs(yshift) > shift_max) | (xrms > rms_max) | (yrms > rms_max):
print 'Shifts out of allowed range. Run again with increased shift_max to accept.'
#return xshift, yshift, rot, scale, xrms, yrms
## Add a small shift that should come out easily with another
## shift iteration
xshift, yshift, rot, scale, xrms, yrms = 2,2,0,1.0,-99,-99
for file in [sci, wht]:
if ('r' in fitgeometry) & rot_only:
xshift, yshift = 0, 0
#apply_offsets(file, [[xshift, yshift, rot, scale]])
from drizzlepac import updatehdr
updatehdr.updatewcs_with_shift(file, sci, wcsname='DRZWCS',
rot=rot,scale=scale,
xsh=xshift, ysh=yshift,
fit=None,
xrms=xrms, yrms = yrms,
verbose=False, force=True, sciext=0)
if '_dr' in sci:
im = pyfits.open(sci)
h = im[0].header
for i in range(h['NDRIZIM']):
flt_str = h['D%03dDATA' %(i+1)]
if 'sci,2' in flt_str:
continue
#
flt_im = flt_str.split('[')[0]
ext = int(flt_str.split('[')[1][:-1].split(',')[1])
updatehdr.updatewcs_with_shift(flt_im, sci, wcsname='GTWEAK', rot=rot, scale=scale, xsh=xshift, ysh=yshift,
fit=None, xrms=xrms, yrms = yrms, verbose=False, force=True, sciext='SCI')
# im = pyfits.open(file, mode='update')
# wcs = pywcs.WCS(im[0].header)
# wcs.rotateCD(-rot)
# wcs.wcs.cd /= scale
# #
# im[0].header['CRPIX1'] += xshift
# im[0].header['CRPIX2'] += yshift
# #
# for i in [0,1]:
# for j in [0,1]:
# im[0].header['CD%d_%d' %(i+1, j+1)] = wcs.wcs.cd[i,j]
# #
# im.flush()
return xshift, yshift, rot, scale, xrms, yrms
def go():
os.chdir('/home/ec2-user/Mosaics')
root = 'j123656p6215'
root = 'j021732m0512'
root = 'j141956p5255'
root = 'j033236m2748'
root = 'j100012p0210'
# Sync needed files for GOODSN
if root == 'j123656p6215':
# os.system('aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/{0}_*dr*fits.gz" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" s3://grizli/Pipeline/ .'.format(root))
# os.system('aws s3 sync --exclude "*" --include "{0}*/Prep/*wcs.log" --include "{0}*/Prep/*shifts.*" s3://grizli/Pipeline/ .'.format(root))
os.system(
'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" s3://grizli/Pipeline/ .'
.format(root))
else:
#os.system('aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*_fl?.*" --include "{0}*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .'.format(root))
os.system(
'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .'
.format(root))
#os.system('files=`find . |grep "dr[cz]_" |grep fits.gz |grep -v "\-ir_dr"`; for file in $files; do echo $file; gunzip -f $file; done')
# Remake catalogs
mos_files = glob.glob('{0}*/Prep/*_dr*sci.fits'.format(root))
mos_files.sort()
thresh = 10
force = False
for i, file in enumerate(mos_files):
root = file.split('_drc_sci')[0].split('_drz_sci')[0]
print(i, root)
cat_file = root + '.cat.fits'
if (not os.path.exists(cat_file)) | (force):
prep.make_SEP_catalog(root=root,
threshold=thresh,
get_background=True,
phot_apertures=[1 * u.arcsec])
os.chdir('Fine/')
os.system('rsync -avz ../{0}*/Prep/*cat.fits .'.format(root))
os.system('ln -s ../{0}*/Prep/*_dr?_sci.fits .'.format(root))
os.system('rsync -avz ../{0}*/Prep/*visits.npy .'.format(root))
os.system('rsync -avz ../{0}*/Prep/*fail* .'.format(root))
visit_files = glob.glob('*visits.npy')
#visit_files = glob.glob('*-tile-*visits.npy')
visit_files.sort()
filt = 'f850lp'
all_visits = []
for f in visit_files:
root_i = f.split('_visits.npy')[0]
cat_file = root_i + '-' + filt + '.cat.fits'
if not os.path.exists(cat_file):
continue
visits, _, _ = np.load(f, allow_pickle=True)
has_failed = False
for v in visits:
has_failed |= os.path.exists(v['product'] + '.failed')
if has_failed:
continue
all_visits.append({'product': root_i + '-' + filt})
# Fine alignment
radec, ref_catalog = prep.get_radec_catalog(ra=189.2316435,
dec=62.249739865958,
product='xxx',
reference_catalogs=['PS1'],
radius=30)
auto_script.fine_alignment(field_root='xxx',
all_visits=all_visits,
gaia_by_date=False,
catalogs=['PS1'],
redrizzle=False,
shift_only=False,
radec=radec,
tol=1.e-3)
##### Done
import numpy as np
import os
import glob
from grizli import prep, utils
from drizzlepac import updatehdr
import astropy.io.fits as pyfits
import astropy.wcs as pywcs
fine_visits, fine_fit = np.load('xxx_fine.npy', allow_pickle=True)
N = len(fine_visits)
trans = np.reshape(fine_fit.x, (N, -1)) #/10.
sh = trans.shape
if sh[1] == 2:
pscl = np.array([10., 10.])
trans = np.hstack([trans / pscl, np.zeros((N, 1)), np.ones((N, 1))])
elif sh[1] == 3:
pscl = np.array([10., 10., 100])
trans = np.hstack([trans / pscl, np.ones((N, 1))])
elif sh[1] == 4:
pscl = np.array([10., 10., 100, 100])
trans = trans / pscl
# Update WCS
# Update direct WCS
for ix, direct in enumerate(fine_visits):
root_i = direct['product'][:-7]
print('\n\n\n\n##### ', ix, root_i, '#######\n\n\n')
# Sync products
os.system(
'aws s3 sync s3://grizli/Pipeline/{0}/Prep/ ./{0}/Prep/ --exclude "*" --include "{0}*drc_sci.fits.gz" --include "*_flc.*fits"'
.format(root_i))
#os.system('aws s3 ls s3://grizli/Pipeline/{0}/Prep/'.format(root_i))
# Just visits and fail
os.system(
'aws s3 sync s3://grizli/Pipeline/{0}/Prep/ ./{0}/Prep/ --exclude "*" --include "*visits.npy" --include "*fail*"'
.format(root_i))
#direct = visits[ix]
out_shift, out_rot = trans[ix, :2], trans[ix, 2]
out_scale = trans[ix, 3]
xyscale = trans[ix, :4]
wcs_ref_file = str(
'{0}/Prep/{0}-f850lp_drc_sci.fits.gz'.format(root_i))
wcs_ref = pywcs.WCS(pyfits.open(wcs_ref_file)[0].header, relax=True)
for file in glob.glob('{0}/Prep/*flc.fits'.format(root_i)):
prep.update_wcs_fits_log(file,
wcs_ref,
xyscale=xyscale,
initialize=False,
replace=('.fits', '.wcslog.fits'),
wcsname='FINE')
updatehdr.updatewcs_with_shift(file,
wcs_ref_file,
xsh=out_shift[0],
ysh=out_shift[1],
rot=out_rot,
scale=out_scale,
wcsname='FINE',
force=True,
reusename=True,
verbose=True,
sciext='SCI')
### Bug in astrodrizzle? Dies if the FLT files don't have MJD-OBS
### keywords
im = pyfits.open(file, mode='update')
im[0].header['MJD-OBS'] = im[0].header['EXPSTART']
im.flush()
## Fields with catalogs
visit_files = glob.glob('j*/Prep/*f814w*visits.npy')
visit_files.sort()
## Mosaic now that they're aligned
# Shift parameters
if False:
os.system(
'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" --include "{0}*/Prep/*expflag*" s3://grizli-v1/Pipeline/ .'
.format(root))
# No catalogs
os.system(
'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*wcs.log" --include "{0}*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .'
.format(root))
# ACS
os.system(
'aws s3 sync --exclude "*" --include "{0}*acswfc*/Prep/*expflag*" --include "{0}*acswfc*/Prep/*fail*" --include "{0}*acswfc*/Prep/*visits.npy" --include "{0}*acswfc*/Prep/*[._]wcs*" --include "{0}*acswfc*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .'
.format(root))
# Shifts
os.system(
'echo "# root visit exp xsh ysh rot scl n xrms yrms" > shifts_results.log'
)
os.system(
'grep "_fl" j*/Prep/*shifts.log | grep -v "\#" | sed "s/:/ /" | sed "s/.Prep./ /g" | sed "s/_shifts.log//" >> shifts_results.log'
)
sh = utils.read_catalog('shifts_results.log')
badsh = (sh['xrms'] > 1) | (sh['yrms'] > 1) #| (sh['n'] < 10)
shift_skip_visits = list(np.unique(sh['visit'][badsh]))
# Flag
os.system('echo "group,x,expm,flag" > expflag_results.csv')
os.system(
'grep "fits" j*/Prep/*expflag*txt | sed "s/:/,/g" | sed "s/.txt/ /" | sed "s/.Prep//" | sed "s/\//,/g" | sed "s/_raw/_flt/" >> expflag_results.csv'
)
expf = utils.read_catalog('expflag_results.csv')
bad_expf = expf['flag'] != 'NORMAL'
exp_visits = np.unique(expf['group'][bad_expf])
exp_visits = []
# wcs
os.system('echo "# dir visit x xs ys rot scl rms n" > shift_wcs.log')
os.system(
'grep " 0 " j*/Prep/*wcs.log | sed "s/.Prep./ /" | sed "s/_wcs.log://" >> shift_wcs.log'
)
shifts = utils.read_catalog('shift_wcs.log')
skip = (shifts['n'] < 7)
skip |= (shifts['xs'] == 0.) | (np.abs(shifts['rot']) > 0.2)
skip_visits = [
'{0}_{1}'.format('_'.join(d.split('_')[:2]), v)
for d, v in zip(shifts['dir'][skip], shifts['visit'][skip])
]
skip_visits += list(exp_visits)
skip_visits += shift_skip_visits
skip_visits = list(np.unique(skip_visits))
skip_keys = ['uds-12-bhm']
skip_keys += ['94s-245.0'] # bad shifts.txt
skip_keys += ['sn2002zx-'] # bad scale
visit_files = glob.glob('j*/Prep/*visits.npy')
#visit_files = glob.glob('*shizels*/Prep/*visits.npy')
all_visits = []
all_groups = []
all_info = []
bucket = 'grizli-v1'
bucket = 'grizli-cosmos-v2'
for file in visit_files:
root_i = os.path.basename(file).split('_visits')[0]
assoc = '_'.join(os.path.basename(file).split('_')[:2])
visits, groups, info = np.load(file)
info['keep'] = False
ic = 0
for v in visits:
failed = glob.glob('{0}/Prep/{1}*fail*'.format(
root_i, v['product']))
if len(failed) > 0:
print(failed)
continue
for k in skip_keys:
if k in v['product']:
print('SKIP', v['product'])
continue
v['product'] = assoc + '_' + v['product']
if v['product'] in skip_visits:
print('SKIP ', v['product'])
continue
v['awspath'] = ['{0}/Pipeline/{1}/Prep'.format(bucket, root_i)
] * len(v['files'])
all_visits.append(v)
for f in v['files']:
info['keep'][info['FILE'] == f] = True
all_info.append(info[info['keep'] == True])
for g in groups:
failed = glob.glob('{0}/Prep/{1}*fail*'.format(
root_i, g['direct']['product']))
if len(failed) > 0:
print(failed)
continue
# for ext in ['direct', 'grism']:
# g[ext]['product'] = assoc+'_'+g[ext]['product']
all_groups.append(g)
import astropy.table
all_info = astropy.table.vstack(all_info)
if root == 'j123656p6215':
out_root = 'gdn-' + root
elif root == 'j021732m0512':
out_root = 'uds-' + root
elif root == 'j141956p5255':
out_root = 'egs-' + root
elif root == 'j033236m2748':
out_root = 'gds-' + root
elif root == 'j100012p0210':
out_root = 'cos-' + root
else:
out_root = 'xxx-' + root
np.save('{0}_visits.npy'.format(out_root),
[all_visits, all_groups, all_info])
if True:
os.system(
'aws s3 cp {0}_visits.npy s3://{1}/Mosaics/ --acl public-read'.
format(out_root, bucket))
all_visits, all_groups, all_info = np.load(
'{0}_visits.npy'.format(out_root))
########
# Remake catalogs and drizzled images
from drizzlepac.astrodrizzle import AstroDrizzle
import numpy as np
import glob
import os
from grizli import prep
all_visits, all_groups, all_info = np.load(
'{0}_visits.npy'.format(out_root))
filters = ['f814w', 'f140w', 'f160w', 'f105w', 'f098m', 'f850lp']
#filters = ['f850lp']
count, products = 0, []
for ii, direct in enumerate(all_visits[::-1]):
filt = direct['product'].split('-')[-1]
prod_files = glob.glob(direct['product'] + '*sci.fits')
if (filt not in filters):
continue
else:
products.append(direct['product'])
if (len(prod_files) > 0):
print('Skip ', direct['product'])
continue
else:
count += 1
print('\n\n\n\n===========\n\n', ii, count, direct['product'],
'\n\n\n========\n')
#break
isACS = '_flc' in direct['files'][0]
if isACS:
bits = 64 + 32 + 256
driz_cr_snr = '3.5 3.0'
driz_cr_scale = '1.2 0.7'
else:
bits = 576 + 256
driz_cr_snr = '8.0 5.0'
driz_cr_scale = '2.5 0.7'
for aws, file in zip(direct['awspath'], direct['files']):
if not os.path.exists(file):
os.system('aws s3 cp s3://{0}/{1} ./'.format(aws, file))
cr_corr = False
AstroDrizzle(direct['files'],
output=direct['product'],
clean=True,
context=False,
preserve=False,
skysub=True,
driz_separate=cr_corr,
driz_sep_wcs=cr_corr,
median=cr_corr,
blot=cr_corr,
driz_cr=cr_corr,
driz_cr_corr=cr_corr,
driz_cr_snr=driz_cr_snr,
driz_cr_scale=driz_cr_scale,
driz_combine=True,
final_bits=bits,
coeffs=True,
resetbits=4096 * cr_corr,
build=False,
final_kernel='point',
final_wht_type='IVM')
# Remake catalog
cat = prep.make_SEP_catalog(root=direct['product'], threshold=5)
os.system('rm {0}*_[wbs]?[gt].fits'.format(direct['product']))
# Remove mosaic
os.system('rm {0}*_sci.fits'.format(direct['product']))
# Remove FLC
if isACS:
for f in direct['files']:
os.remove(f)
# Make mosaics
kwargs = auto_script.get_yml_parameters()
mos_args = {
'mosaic_args':
kwargs['mosaic_args'],
'fix_stars':
kwargs['visit_prep_args']['fix_stars'],
'mask_spikes':
kwargs['mask_spikes'],
'skip_single_optical_visits':
kwargs['preprocess_args']['skip_single_optical_visits']
}
#mos_args['mosaic_args']['ir_filters'] = ['F140W']
#mos_args['mosaic_args']['optical_filters'] = ['F814W']
mos_args['mosaic_args']['fill_mosaics'] = False
mos_args['mosaic_args']['half_optical_pixscale'] = True
mos_args['mosaic_args']['kernel'] = 'square'
mos_args['mosaic_args']['pixfrac'] = 0.33
mos_args['mosaic_args']['wcs_params']['pixel_scale'] = 0.1
if root == 'j123656p6215':
mos_args['mosaic_args']['kernel'] = 'point'
mos_args['mosaic_args']['pixfrac'] = 0.33
mos_args['mosaic_args']['wcs_params']['pixel_scale'] = 0.1
#mos_args['mosaic_args']['wcs_params']['filters'] = ['F140W']
os.system('ln -s j*/Prep/*_fl?.fits .')
auto_script.make_combined_mosaics(out_root, **mos_args)
