def applyShiftsSA(visit): sa_out = '%s_simplematch.out' % visit drzs = {} print('Reading %s...' % sa_out) for i in [j.split() for j in open(sa_out).readlines()]: drz = i[0].replace('_sa.cat', '.fits') drzs[drz] = [float(i[1]), float(i[2]), float(i[3])] print("Applying shifts...") with open('sa_shifts.txt', 'a') as sas: for d, s in drzs.items(): if os.path.exists(d): if d[:6] == visit: if s: dx, dy, dt = s wcs = HSTWCS(fits.open(d)) dxp = round(dx / wcs.pscale, 3) dyp = round(dy / wcs.pscale, 3) dtp = round(dt, 3) offsets = [d, dxp, dyp, dtp] sas.write('%s %.3f %.3f %.3f\n' % (d, dxp, dyp, dtp)) print(d, dxp, dyp, dtp) updatehdr.updatewcs_with_shift(d, d, wcsname='DRZWCS', xsh=dxp, ysh=dyp, rot=dtp, scale=1.0, force=True)
def shift_wcs(im): try: im_wcs = wcs.WCS(fits.getheader(im, 1)) except: raise AssertionError('COULDN\'T FIND WCS FOR {}'.format(im)) if im_wcs.wcs.name == 'HSC': print '{} already aligned to HSC, skipping'.format(im) return f = [ln.strip('\n') for ln in open('hsc_shifts.txt').readlines()][-1] xsh, ysh, rot, scl, xrms, yrms = f.split()[1:] ref = 'hsc_shifts_wcs.fits' print im try: updatehdr.updatewcs_with_shift(im, ref, xsh=xsh, ysh=ysh, rot=rot, scale=scl, wcsname='HSC', force=False, sciext='SCI') tweakback.tweakback(im, force=False, origwcs='DRZWCS', newname='HSC', wcsname='HSC') except: print 'COULDN\'T UPDATE {}, ERRORING'.format(im) raise
def shift_wcs(im): try: im_wcs = wcs.WCS(fits.getheader(im,1)) except: raise AssertionError('COULDN\'T FIND WCS FOR {}'.format(im)) if im_wcs.wcs.name == 'HSC': print '{} already aligned to HSC, skipping'.format(im) return f = [ln.strip('\n') for ln in open('hsc_shifts.txt').readlines()][-1] xsh, ysh, rot, scl, xrms, yrms = f.split()[1:] ref = 'hsc_shifts_wcs.fits' print im try: updatehdr.updatewcs_with_shift(im,ref,xsh=xsh,ysh=ysh,rot=rot,scale=scl,wcsname='HSC',force=False,sciext='SCI') tweakback.tweakback(im,force=False,origwcs='DRZWCS',newname='HSC',wcsname='HSC') except: print 'COULDN\'T UPDATE {}, ERRORING'.format(im) raise
def apply_shift(self, database, input_image="", suffix='geo', force_north_up=True): # Apply the shifts to input image WCS header keywords # Once the database has been calculated, one can use this to shift many # other images. # First read the shifts (in arcsec) and rotation (in degrees) xshift, yshift, xrot = self.read_shifts(database) # Then calculate the shifts in INPUT IMAGE PIXELS xshift_pix = xshift / self.input_pixscale yshift_pix = yshift / self.input_pixscale print "xshift_pix, yshift_pix", xshift_pix, yshift_pix # Now call updatehdr.updatewcs_with_shift if not input_image: input_image = self.input_image output_image = os.path.splitext(input_image)[0] + '_%s.fits' % suffix if os.path.exists(output_image): print "Removing %s..." % output_image os.remove(output_image) os.system('cp %s %s' % (input_image, output_image)) updatehdr.updatewcs_with_shift(output_image, input_image, rot=xrot, scale=1.0, xsh=-xshift_pix, ysh=-yshift_pix, force=True, verbose=True) if force_north_up: h = pyfits.open(output_image, mode='update') pixscale = np.sqrt(h[0].header['cd1_1']**2 + h[0].header['cd1_2']**2) # in degrees h[0].header['cd1_1'] = np.sign(h[0].header['cd1_1']) * pixscale h[0].header['cd1_2'] = 0. h[0].header['cd2_1'] = 0. h[0].header['cd2_2'] = np.sign(h[0].header['cd2_2']) * pixscale h[0].header['orientat'] = 0. h.flush() h.close()
def refineShiftMCMC(drzfile): """ Refine shifts using MCMC """ dsn = drzfile[:9] wcs = HSTWCS(fits.open(drzfile)) wcsn = fits.getval(drzfile, 'wcsname') try: refcat = np.loadtxt('%s_drz_sci_sa_ref_match.cat' % dsn, usecols=(1, 2)) imgcat = np.loadtxt('%s_drz_sci_sa_match.cat' % dsn, usecols=(1, 2)) refcatw = wcs.all_world2pix(refcat, 1) imgcatw = wcs.all_world2pix(imgcat, 1) ox, oy = wcs.wcs.crpix.tolist() offset, err = mcmcShifts.findOffsetMCMC(imgcatw, refcatw, maxShift=(10, 10, 0.3), rotOrigin=(ox, oy), precision=0.01, visualize=False) print(drzfile, offset, err) dxp, dyp, dtp = offset updatehdr.updatewcs_with_shift(drzfile, drzfile, wcsname='DRZWCS', xsh=dxp, ysh=dyp, rot=dtp, scale=1.0, force=True) return offset except UserWarning: pass
def get_align_to_subaru(sci='M0416_Ks_c1_mp_avg.fits', wht='M0416_Ks_c1_mp_exp.fits', field='', clean=True, toler=3, verbose=False, fitgeometry='shift', shift_max=20, rms_max=1.1, rot_max=2, rot_only=True, THRESH=2, align_data=None): """ Align HAWK-I images to the FF Subaru astrometric reference catalogs """ #sci='M0416_Ks_c1_mp_avg.fits'; wht='M0416_Ks_c1_mp_exp.fits' ### Make object catalog se = threedhst.sex.SExtractor() se.aXeParams() se.copyConvFile() se.overwrite = True se.options['CHECKIMAGE_TYPE'] = 'NONE' if wht is None: se.options['WEIGHT_TYPE'] = 'NONE' else: se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT' se.options['WEIGHT_IMAGE'] = wht se.options['FILTER'] = 'Y' se.options['DETECT_THRESH'] = '%d' % (THRESH) se.options['ANALYSIS_THRESH'] = '%d' % (THRESH) se.options['MAG_ZEROPOINT'] = '26.0' #### Run SExtractor on direct and alignment images ## direct image se.options['CATALOG_NAME'] = 'direct.cat' status = se.sextractImage(sci) threedhst.sex.sexcatRegions('direct.cat', 'direct.reg', format=2) directCat = threedhst.sex.mySexCat('direct.cat') #### Get the X/Y coords of the reference catalog #head = pyfits.getheader(sci, 0) #wcs = pywcs.WCS(head) if 'M0416' in sci: ra_list, dec_list, mag = np.loadtxt( os.getenv('HAWKI') + '/FrontierFields/HST/hlsp_frontier_subaru_suprimecam_macs0416-astrom_R_v1_cat.txt', unpack=True) if ('c4' in sci): ra_list, dec_list, mag = np.loadtxt( os.getenv('HAWKI') + '/FrontierFields/HST/M0416/macs0416_f814w_radec.cat', unpack=True) # if 'M0717' in sci: ra_list, dec_list, mag = np.loadtxt('subaru.radec', unpack=True) if ('M1149' in sci) | (field == 'M1149'): ra_list, dec_list, mag = np.loadtxt( '/Users/brammer/Research/VLT/HAWKI/MACS1149/hlsp_frontier_subaru_suprimecam_macs1149-astrom_R_v1_cat.txt', unpack=True) if 'A2744' in sci: ra_list, dec_list, mag = np.loadtxt( os.getenv('HAWKI') + '/FrontierFields/HST/hlsp_frontier_subaru_suprimecam_abell2744-astrom_i_v1_cat.txt', unpack=True) if ('c1' in sci) | ('c4' in sci): ra_list, dec_list, mag = np.loadtxt( os.getenv('HAWKI') + '/FrontierFields/HST/abell2744_f814w_radec.cat', unpack=True) if align_data is not None: ra_list, dec_list, mag = align_data im = pyfits.open(sci) print sci sh = im[0].shape head = im[0].header head['CUNIT1'] = 'deg' head['CUNIT2'] = 'deg' wcs = pywcs.WCS(head) x_image, y_image = wcs.wcs_sky2pix(ra_list, dec_list, 1) try: x_image, y_image = wcs.wcs_sky2pix(ra_list, dec_list, 1) except: x_image, y_image = wcs.wcs_world2pix(ra_list, dec_list, 1) ok = (x_image > 0) & (y_image > 0) & (x_image < sh[1]) & (y_image < sh[1]) x_image, y_image = x_image[ok], y_image[ok] fpr = open('align.reg', 'w') fpr.write('image\n') for i in range(ok.sum()): fpr.write('circle(%.6f, %.6f,0.3") # color=magenta\n' % (x_image[i], y_image[i])) fpr.close() # x_image, y_image = [], [] # # for ra, dec in zip(ra_list, dec_list): # x, y = wcs.wcs_sky2pix([[ra, dec]], 1)[0] # if (x > 0) & (y > 0) & (x < sh[1]) & (y < sh[1]): # x_image.append(x) # y_image.append(y) alignCat = catIO.EmptyCat() alignCat['X_IMAGE'] = np.array(x_image) alignCat['Y_IMAGE'] = np.array(y_image) xshift = 0 yshift = 0 rot = 0 scale = 1. xrms = 2 yrms = 2 NITER = 5 IT = 0 while (IT < NITER): IT = IT + 1 #### Get x,y coordinates of detected objects ## direct image fp = open('direct.xy', 'w') for i in range(len(directCat.X_IMAGE)): fp.write('%s %s\n' % (directCat.X_IMAGE[i], directCat.Y_IMAGE[i])) fp.close() ## alignment image fp = open('align.xy', 'w') for i in range(len(alignCat.X_IMAGE)): fp.write('%s %s\n' % (np.float(alignCat.X_IMAGE[i]) + xshift, np.float(alignCat.Y_IMAGE[i]) + yshift)) fp.close() iraf.flpr() iraf.flpr() iraf.flpr() #### iraf.xyxymatch to find matches between the two catalogs pow = toler * 1. try: os.remove('align.match') except: pass status1 = iraf.xyxymatch(input="direct.xy", reference="align.xy", output="align.match", tolerance=2**pow, separation=0, verbose=iraf.yes, Stdout=1) nmatch = 0 while status1[-1].startswith('0') | (nmatch < 10) | (float( status1[-3].split()[1]) > 40): pow += 1 os.remove('align.match') status1 = iraf.xyxymatch(input="direct.xy", reference="align.xy", output="align.match", tolerance=2**pow, separation=0, verbose=iraf.yes, Stdout=1) # nmatch = 0 for line in open('align.match').xreadlines(): nmatch += 1 if verbose: for line in status1: print line #### Compute shifts with iraf.geomap iraf.flpr() iraf.flpr() iraf.flpr() try: os.remove("align.map") except: pass status2 = iraf.geomap(input="align.match", database="align.map", fitgeometry=fitgeometry, interactive=iraf.no, xmin=iraf.INDEF, xmax=iraf.INDEF, ymin=iraf.INDEF, ymax=iraf.INDEF, maxiter=10, reject=2.0, Stdout=1) if verbose: for line in status2: print line #fp = open(root+'.iraf.log','a') #fp.writelines(status1) #fp.writelines(status2) #fp.close() #### Parse geomap.output fp = open("align.map", "r") for line in fp.readlines(): spl = line.split() if spl[0].startswith('xshift'): xshift += float(spl[1]) if spl[0].startswith('yshift'): yshift += float(spl[1]) if spl[0].startswith('xrotation'): rot = float(spl[1]) if spl[0].startswith('xmag'): scale = float(spl[1]) if spl[0].startswith('xrms'): xrms = float(spl[1]) if spl[0].startswith('yrms'): yrms = float(spl[1]) fp.close() #os.system('wc align.match') print 'Shift iteration #%d, xshift=%f, yshift=%f, rot=%f, scl=%f (rms: %5.2f,%5.2f)' % ( IT, xshift, yshift, rot, scale, xrms, yrms) os.system( 'cat align.match | grep -v "\#" | grep [0-9] | awk \'{print "circle(", $1, ",", $2, ",4) # color=green"}\' > d.reg' ) os.system( 'cat align.match | grep -v "\#" | grep [0-9] | awk \'{print "circle(", $3, ",", $4, ",4) # color=magenta"}\' > a.reg' ) shutil.copy('align.map', sci.replace('.fits', '.align.map')) shutil.copy('align.match', sci.replace('.fits', '.align.match')) #### Cleanup if clean: rmfiles = [ 'align.cat', 'align.map', 'align.match', 'align.reg', 'align.xy', 'direct.cat', 'direct.reg', 'direct.xy' ] for file in rmfiles: try: os.remove(file) except: pass fp = open(sci.replace('.fits', '.align.info'), 'w') fp.write('# image xshift yshift rot scale xrms yrms\n') fp.write('%s %.3f %.3f %.4f %.4f %.3f %.3f\n' % (sci, xshift, yshift, rot, scale, xrms, yrms)) if (np.abs(xshift) > shift_max) | (np.abs(yshift) > shift_max) | ( xrms > rms_max) | (yrms > rms_max): print 'Shifts out of allowed range. Run again with increased shift_max to accept.' #return xshift, yshift, rot, scale, xrms, yrms ## Add a small shift that should come out easily with another ## shift iteration xshift, yshift, rot, scale, xrms, yrms = 2, 2, 0, 1.0, -99, -99 for file in [sci, wht]: if ('r' in fitgeometry) & rot_only: xshift, yshift = 0, 0 #apply_offsets(file, [[xshift, yshift, rot, scale]]) from drizzlepac import updatehdr updatehdr.updatewcs_with_shift(file, sci, wcsname='DRZWCS', rot=rot, scale=scale, xsh=xshift, ysh=yshift, fit=None, xrms=xrms, yrms=yrms, verbose=False, force=True, sciext=0) if '_dr' in sci: im = pyfits.open(sci) h = im[0].header for i in range(h['NDRIZIM']): flt_str = h['D%03dDATA' % (i + 1)] if 'sci,2' in flt_str: continue # flt_im = flt_str.split('[')[0] ext = int(flt_str.split('[')[1][:-1].split(',')[1]) updatehdr.updatewcs_with_shift(flt_im, sci, wcsname='GTWEAK', rot=rot, scale=scale, xsh=xshift, ysh=yshift, fit=None, xrms=xrms, yrms=yrms, verbose=False, force=True, sciext='SCI') # im = pyfits.open(file, mode='update') # wcs = pywcs.WCS(im[0].header) # wcs.rotateCD(-rot) # wcs.wcs.cd /= scale # # # im[0].header['CRPIX1'] += xshift # im[0].header['CRPIX2'] += yshift # # # for i in [0,1]: # for j in [0,1]: # im[0].header['CD%d_%d' %(i+1, j+1)] = wcs.wcs.cd[i,j] # # # im.flush() return xshift, yshift, rot, scale, xrms, yrms
def auto_run(root='j023507-040202', flag_global_crs=False): import os import glob import numpy as np import astropy.io.fits as pyfits import astropy.wcs as pywcs from drizzlepac import updatehdr from stwcs import updatewcs from grizli import utils, prep from grizli.pipeline import auto_script utils.set_warnings() visit_file = '{0}_visits.npy'.format(root) visits, all_groups, info = np.load(visit_file) # Something wrong with some files with bad shifts, reset wcs for visit in visits: for file in visit['files']: utils.fetch_hst_calibs( file, calib_types=['IDCTAB', 'NPOLFILE', 'IMPHTTAB']) updatewcs.updatewcs(file, verbose=True, use_db=False) # Apply shifts shift_log = '{0}_shifts.log'.format(visit['product']) if os.path.exists(shift_log): sh = utils.read_catalog(shift_log) flt0 = pyfits.open(sh['flt'][0]) wcs_ref = pywcs.WCS(flt0['SCI', 1].header, fobj=flt0, relax=True) shift_dict = {} for i in range(len(sh)): shift_dict[sh['flt'][i]] = [sh['xshift'][i], sh['yshift'][i]] prep.apply_tweak_shifts(wcs_ref, shift_dict, grism_matches={}, verbose=False) # Redrizzle mosaics prep.drizzle_overlaps(visits, check_overlaps=False, skysub=False, static=False, pixfrac=0.5, scale=None, final_wcs=False, fetch_flats=False, final_rot=None, include_saturated=True) ####### Alignment os.system('rm *wcs.*') # Radec master_radec = '{0}/../../{1}_master.radec'.format(os.getcwd(), root) if not os.path.exists(master_radec): master_radec = None ref_catalog = 'USER' if root.startswith('cos-'): hsc = '{0}/../../{1}'.format(os.getcwd(), 'hsc-udeep-i25_corr_cosmos.radec') if os.path.exists(hsc): master_radec = hsc ref_catalog = 'HSC' elif root.startswith('uds-'): hsc = '{0}/../../{1}'.format(os.getcwd(), 'hsc-udeep-sxds_corr_uds.radec') if os.path.exists(hsc): master_radec = hsc ref_catalog = 'HSC' parent_radec = '{0}/../../{1}_parent.radec'.format(os.getcwd(), root) if not os.path.exists(parent_radec): parent_radec = None if master_radec is not None: radec = master_radec elif parent_radec is not None: radec = parent_radec else: radec = None if radec is None: needs_gaia = True else: needs_gaia = False REFERENCE = 'GAIA' REFERENCE = 'PS1' print('master RADEC file: ', radec) thresh = 2.5 for visit in visits: # Clean catalogs files = glob.glob('{0}.*'.format(visit['product'])) for file in files: os.remove(file) # Generate GAIA alignment catalog at the observation epoch clip = 120 clip = -1 if needs_gaia: flt = pyfits.open(visit['files'][0]) h = flt['SCI', 1].header ra_i, dec_i = h['CRVAL1'], h['CRVAL2'] radec, ref_catalog = prep.get_radec_catalog( ra=ra_i, dec=dec_i, product=visit['product'], date=flt[0].header['EXPSTART'], date_format='mjd', reference_catalogs=[REFERENCE], radius=5.) flt.close() if REFERENCE == 'GAIA': mag_limits = [16, 20] else: mag_limits = [18, 22] #clip = 50 if '_flc' in visit['files'][0]: triangle_size_limit = [5, 4000 * np.sqrt(2)] else: triangle_size_limit = [5, 1300] else: mag_limits = [19, 23] triangle_size_limit = [5, 1300] # Remake catalogs cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh) # Redo alignment try: print('XXX clip', clip, mag_limits, triangle_size_limit) result = prep.align_drizzled_image( root=visit['product'], radec=radec, mag_limits=mag_limits, simple=False, max_err_percentile=80, clip=clip, outlier_threshold=5, rms_limit=2.5, triangle_size_limit=triangle_size_limit) except: print('First align failed! Relax parameters') try: result = prep.align_drizzled_image( root=visit['product'], radec=radec, mag_limits=[10, 20], simple=False, max_err_percentile=99, clip=160, outlier_threshold=20, rms_limit=2.5, triangle_size_limit=triangle_size_limit) except: try: result = prep.align_drizzled_image( root=visit['product'], radec=radec, mag_limits=[10, 20], simple=False, max_err_percentile=99, clip=160, outlier_threshold=40, rms_limit=2.5, triangle_size_limit=triangle_size_limit) except: radec = '{0}_ps1.radec'.format(visit['product']) ref_catalog = 'PS1' result = prep.align_drizzled_image( root=visit['product'], radec=radec, mag_limits=mag_limits, simple=False, max_err_percentile=80, clip=120, outlier_threshold=5, rms_limit=2.5, triangle_size_limit=triangle_size_limit) #continue orig_wcs, drz_wcs, out_shift, out_rot, out_scale = result # Propagate shifts for file in visit['files']: updatehdr.updatewcs_with_shift(file, str('{0}_wcs.fits'.format( visit['product'])), xsh=out_shift[0], ysh=out_shift[1], rot=out_rot, scale=out_scale, wcsname=ref_catalog, force=True, reusename=True, verbose=True, sciext='SCI') ### Bug in astrodrizzle? Dies if the FLT files don't have MJD-OBS ### keywords im = pyfits.open(file, mode='update') im[0].header['MJD-OBS'] = im[0].header['EXPSTART'] im.flush() # Redrizzle mosaics again including new shifts prep.drizzle_overlaps(visits, check_overlaps=False, skysub=False, static=False, pixfrac=0.8, scale=None, final_wcs=False, fetch_flats=False, final_rot=None) # Remake catalogs thresh = 2.5 for visit in visits: # Remake catalogs cat = prep.make_SEP_catalog(root=visit['product'], threshold=thresh) prep.table_to_regions(cat, '{0}.cat.reg'.format(visit['product'])) prep.table_to_radec(cat, '{0}.cat.radec'.format(visit['product'])) # Update visits file v = auto_script.get_visit_exposure_footprints(visit_file=visit_file, check_paths=['./', '../RAW'], simplify=1.e-6) if flag_global_crs: # Assume everything at same orient pass if False: # Mosaic auto_script.drizzle_overlaps(root, filters=['F160W'], min_nexp=1, pixfrac=0.8, scale=0.1, make_combined=False, ref_image=None, static=False)
def SingleStarReg(imfile, ra, dec, wcsname='SINGLESTAR', computesig=False, refim=None, threshmin=0.5, peakmin=None, peakmax=None, searchrad=5.0, nsigma=1.5, fluxmin=None, fluxmax=None, verbose=True, clobber=True): """ Update the WCS of imfile so that it matches the WCS of the refimfile, using a single star to define the necessary shift. If xy or xyref are provided, then assume the star is located within searchrad arcsec of that position in the imfile or the refimfile, respectively. If either of those pixel coordinates are not provided, then use the brightest star in the image. """ # noinspection PyUnresolvedReferences from numpy import deg2rad, sqrt, cos, where, median from astropy.io import ascii from drizzlepac.updatehdr import updatewcs_with_shift from numpy import unique if refim is None: refim = imfile if computesig == False: skysigma = getskysigma(imfile) if verbose: print(("sndrizipipe.register.SingleStarReg: " " Manually computed sky sigma for %s as %.5e" % (imfile, skysigma))) else: skysigma = 0.0 # locate stars in imfile, pick out the brightest one topdir = os.path.abspath('.') imfiledir = os.path.dirname(os.path.abspath(imfile)) imfilebase = os.path.basename(imfile) os.chdir(imfiledir) # Iterate the source-finding algorithm with progressively smaller threshold # values. This helps to ensure that we correctly locate the single bright # source in the image xycatfile = None threshold = 200. while threshold >= threshmin: try: xycatfile = mkSourceCatalog(imfilebase, computesig=computesig, skysigma=skysigma, nsigma=nsigma, threshold=threshold, peakmin=peakmin, peakmax=peakmax, fluxmin=fluxmin, fluxmax=fluxmax)[0] # The source finder succeeded! radeccatfile = xycatfile.replace('xy.coo', 'radec.coo') break except NameError: # the source finder failed, try again with a lower threshold threshold /= 2. continue if xycatfile is None: print(( "Failed to generate a clean source catalog for %s"%imfile + \ " using threshmin = %.3f"%threshmin )) import pdb pdb.set_trace() raise RuntimeError( "Failed to generate a clean source catalog for %s"%imfile + \ " using threshmin = %.3f"%threshmin ) xycat = ascii.read(xycatfile) radeccat = ascii.read(radeccatfile) if verbose: print(("Located %i sources with threshold=%.1f sigma" % (len(xycat), threshold))) os.chdir(topdir) # compute the approximate separation in arcsec from the target ra,dec # to each of the detected sources, then limit to those within searchrad rasrc, decsrc = radeccat['col1'], radeccat['col2'] darcsec = sqrt(((rasrc - ra) * cos(deg2rad(dec)))**2 + (decsrc - dec)**2) * 3600. inear = where(darcsec <= searchrad)[0] if verbose: print((" %i of these sources are within %.1f arcsec of the target" % (len(inear), searchrad))) # identify the brightest source within searchrad arcsec of the target ibrightest = inear[xycat['col3'][inear].argmax()] xfnd, yfnd = [xycat['col1'][ibrightest], xycat['col2'][ibrightest]] if verbose: brightratio = xycat['col3'][ibrightest] / median(xycat['col3'][inear]) print(( " The brightest of these sources is %.1fx brighter than the median." % brightratio)) # The TweakReg imagefind algorithm sometimes misses the true center # catastrophically. Here we use a centroiding algorithm to re-center the # position on the star, checking for a large offset. # Note that we are using numpy arrays, so the origin is (0,0) and we need # to correct the cntrd output to the (1,1) origin used by pyfits and drizzlepac. imdat = pyfits.getdata(imfile) fwhmpix = getfwhmpix(imfile) xcntrd, ycntrd = cntrd(imdat, xfnd, yfnd, fwhmpix) if xcntrd == -1: if verbose: print('Recentering within a 5-pixel box') xcntrd, ycntrd = cntrd(imdat, xfnd, yfnd, fwhmpix, extendbox=5) assert ((xcntrd > 0) & (ycntrd > 0)), "Centroid recentering failed for %s" % imfile xcntrd += 1 ycntrd += 1 dxcntrd = xfnd - xcntrd dycntrd = yfnd - ycntrd if verbose > 9: # plot the centroid shift and save a .png image from matplotlib import pylab as pl from matplotlib import cm pl.clf() vmin = imdat[ycntrd - 10:ycntrd + 10, xcntrd - 10:xcntrd + 10].min() vmax = imdat[ycntrd - 10:ycntrd + 10, xcntrd - 10:xcntrd + 10].max() pl.imshow(imdat, cmap=cm.Greys, vmin=vmin, vmax=vmax) pl.plot(xfnd - 1, yfnd - 1, 'r+', ms=12, mew=1.5, label='drizzlepac.ndfind source position: %.3f, %.3f' % (xfnd, yfnd)) pl.plot(xcntrd - 1, ycntrd - 1, 'gx', ms=12, mew=1.5, label='register.cntrd recentered position: %.3f, %.3f' % (xcntrd, ycntrd)) pl.title("Centroiding shift : dx = %.3f dy = %.3f" % (dxcntrd, dycntrd)) ax = pl.gca() ax.set_xlim(xcntrd - 10, xcntrd + 10) ax.set_ylim(ycntrd - 10, ycntrd + 10) pl.colorbar() ax.set_xlabel('X (pixels)') ax.set_ylabel('Y (pixels)') ax.legend(loc='upper right', numpoints=1, frameon=False) pl.draw() outpng = imfile.replace('.fits', '_recenter.png') pl.savefig(outpng) print(("Saved a recentering image as " + outpng)) # locate the appropriate extensions for updating hdulist = pyfits.open(imfile) sciextlist = [hdu.name for hdu in hdulist if 'WCSAXES' in hdu.header] # convert the target position from ra,dec to x,y if len(sciextlist) > 0: imwcs = stwcs.wcsutil.HSTWCS(hdulist, ext=(sciextlist[0], 1)) else: sciextlist = ['PRIMARY'] imwcs = stwcs.wcsutil.HSTWCS(hdulist, ext=None) xref, yref = imwcs.wcs_world2pix(ra, dec, 1) # If the new centroid position differs substantially from the original # ndfind position, then update the found source position # (i.e. in cases of catastrophic ndfind failure) if (abs(dxcntrd) > 0.5) or (abs(dycntrd) > 0.5): xfnd = xcntrd yfnd = ycntrd # compute the pixel shift from the xy position found in the image # to the reference (target) xy position xshift = xfnd - xref yshift = yfnd - yref # apply that shift to the image wcs for sciext in unique(sciextlist): print(("Updating %s ext %s with xshift,yshift = %.5f %.5f" % (imfile, sciext, xshift, yshift))) updatewcs_with_shift(imfile, refim, wcsname=wcsname, rot=0.0, scale=1.0, xsh=xshift, ysh=yshift, fit=None, xrms=None, yrms=None, verbose=verbose, force=clobber, sciext=sciext) hdulist.close() return (wcsname)
def SingleStarReg( imfile, ra, dec, wcsname='SINGLESTAR', computesig=False, refim=None, threshmin=0.5, peakmin=None, peakmax=None, searchrad=5.0, nsigma=1.5, fluxmin=None, fluxmax=None, verbose=True, clobber=True ): """ Update the WCS of imfile so that it matches the WCS of the refimfile, using a single star to define the necessary shift. If xy or xyref are provided, then assume the star is located within searchrad arcsec of that position in the imfile or the refimfile, respectively. If either of those pixel coordinates are not provided, then use the brightest star in the image. """ # noinspection PyUnresolvedReferences from numpy import deg2rad, sqrt, cos, where, median from astropy.io import ascii from drizzlepac.updatehdr import updatewcs_with_shift from numpy import unique if refim is None : refim = imfile if computesig==False : skysigma = getskysigma( imfile ) if verbose : print(("sndrizipipe.register.SingleStarReg: " " Manually computed sky sigma for %s as %.5e"%(imfile,skysigma))) else : skysigma=0.0 # locate stars in imfile, pick out the brightest one topdir = os.path.abspath( '.' ) imfiledir = os.path.dirname( os.path.abspath(imfile) ) imfilebase = os.path.basename( imfile ) os.chdir( imfiledir ) # Iterate the source-finding algorithm with progressively smaller threshold # values. This helps to ensure that we correctly locate the single bright # source in the image xycatfile = None threshold = 200. while threshold >= threshmin : try : xycatfile = mkSourceCatalog( imfilebase, computesig=computesig, skysigma=skysigma, nsigma=nsigma, threshold=threshold, peakmin=peakmin, peakmax=peakmax, fluxmin=fluxmin, fluxmax= fluxmax )[0] # The source finder succeeded! radeccatfile = xycatfile.replace('xy.coo','radec.coo') break except NameError : # the source finder failed, try again with a lower threshold threshold /= 2. continue if xycatfile is None : print(( "Failed to generate a clean source catalog for %s"%imfile + \ " using threshmin = %.3f"%threshmin )) import pdb; pdb.set_trace() raise RuntimeError( "Failed to generate a clean source catalog for %s"%imfile + \ " using threshmin = %.3f"%threshmin ) xycat = ascii.read( xycatfile ) radeccat = ascii.read( radeccatfile ) if verbose : print(("Located %i sources with threshold=%.1f sigma"%(len(xycat),threshold))) os.chdir( topdir ) # compute the approximate separation in arcsec from the target ra,dec # to each of the detected sources, then limit to those within searchrad rasrc, decsrc = radeccat['col1'], radeccat['col2'] darcsec = sqrt( ((rasrc-ra)*cos(deg2rad(dec)))**2 + (decsrc-dec)**2 ) * 3600. inear = where( darcsec <= searchrad )[0] if verbose : print((" %i of these sources are within %.1f arcsec of the target"%(len(inear),searchrad))) # identify the brightest source within searchrad arcsec of the target ibrightest = inear[ xycat['col3'][inear].argmax() ] xfnd,yfnd = [ xycat['col1'][ibrightest], xycat['col2'][ibrightest] ] if verbose : brightratio = xycat['col3'][ibrightest] / median(xycat['col3'][inear]) print((" The brightest of these sources is %.1fx brighter than the median."%brightratio)) # The TweakReg imagefind algorithm sometimes misses the true center # catastrophically. Here we use a centroiding algorithm to re-center the # position on the star, checking for a large offset. # Note that we are using numpy arrays, so the origin is (0,0) and we need # to correct the cntrd output to the (1,1) origin used by pyfits and drizzlepac. imdat = pyfits.getdata( imfile ) fwhmpix = getfwhmpix( imfile ) xcntrd, ycntrd = cntrd( imdat, xfnd, yfnd, fwhmpix ) if xcntrd==-1 : if verbose : print('Recentering within a 5-pixel box') xcntrd, ycntrd = cntrd( imdat, xfnd, yfnd, fwhmpix, extendbox=5 ) assert ((xcntrd>0) & (ycntrd>0)), "Centroid recentering failed for %s"%imfile xcntrd += 1 ycntrd += 1 dxcntrd = xfnd-xcntrd dycntrd = yfnd-ycntrd if verbose >9 : # plot the centroid shift and save a .png image from matplotlib import pylab as pl from matplotlib import cm pl.clf() vmin = imdat[ycntrd-10:ycntrd+10,xcntrd-10:xcntrd+10].min() vmax = imdat[ycntrd-10:ycntrd+10,xcntrd-10:xcntrd+10].max() pl.imshow( imdat, cmap=cm.Greys, vmin=vmin, vmax=vmax) pl.plot( xfnd-1, yfnd-1, 'r+', ms=12, mew=1.5, label='drizzlepac.ndfind source position: %.3f, %.3f'%(xfnd,yfnd) ) pl.plot( xcntrd-1, ycntrd-1, 'gx', ms=12, mew=1.5, label='register.cntrd recentered position: %.3f, %.3f'%(xcntrd,ycntrd) ) pl.title( "Centroiding shift : dx = %.3f dy = %.3f"%(dxcntrd,dycntrd) ) ax = pl.gca() ax.set_xlim( xcntrd-10, xcntrd+10 ) ax.set_ylim( ycntrd-10, ycntrd+10 ) pl.colorbar() ax.set_xlabel('X (pixels)') ax.set_ylabel('Y (pixels)') ax.legend( loc='upper right', numpoints=1, frameon=False ) pl.draw() outpng = imfile.replace('.fits','_recenter.png') pl.savefig(outpng) print(("Saved a recentering image as " + outpng )) # locate the appropriate extensions for updating hdulist = pyfits.open( imfile ) sciextlist = [ hdu.name for hdu in hdulist if 'WCSAXES' in hdu.header ] # convert the target position from ra,dec to x,y if len(sciextlist)>0: imwcs = stwcs.wcsutil.HSTWCS( hdulist, ext=(sciextlist[0],1) ) else: sciextlist = ['PRIMARY'] imwcs = stwcs.wcsutil.HSTWCS( hdulist, ext=None ) xref, yref = imwcs.wcs_world2pix( ra, dec , 1) # If the new centroid position differs substantially from the original # ndfind position, then update the found source position # (i.e. in cases of catastrophic ndfind failure) if (abs(dxcntrd)>0.5) or (abs(dycntrd)>0.5) : xfnd = xcntrd yfnd = ycntrd # compute the pixel shift from the xy position found in the image # to the reference (target) xy position xshift = xfnd - xref yshift = yfnd - yref # apply that shift to the image wcs for sciext in unique(sciextlist): print(("Updating %s ext %s with xshift,yshift = %.5f %.5f"%( imfile,sciext, xshift, yshift ))) updatewcs_with_shift(imfile, refim, wcsname=wcsname, rot=0.0, scale=1.0, xsh=xshift, ysh=yshift, fit=None, xrms=None, yrms=None, verbose=verbose, force=clobber, sciext=sciext ) hdulist.close() return( wcsname )
def get_align_to_subaru(sci='M0416_Ks_c1_mp_avg.fits', wht='M0416_Ks_c1_mp_exp.fits', field='', clean=True, toler=3, verbose=False, fitgeometry='shift', shift_max=20, rms_max=1.1, rot_max=2, rot_only=True, THRESH=2, align_data=None): """ Align HAWK-I images to the FF Subaru astrometric reference catalogs """ #sci='M0416_Ks_c1_mp_avg.fits'; wht='M0416_Ks_c1_mp_exp.fits' ### Make object catalog se = threedhst.sex.SExtractor() se.aXeParams() se.copyConvFile() se.overwrite = True se.options['CHECKIMAGE_TYPE'] = 'NONE' if wht is None: se.options['WEIGHT_TYPE'] = 'NONE' else: se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT' se.options['WEIGHT_IMAGE'] = wht se.options['FILTER'] = 'Y' se.options['DETECT_THRESH'] = '%d' %(THRESH) se.options['ANALYSIS_THRESH'] = '%d' %(THRESH) se.options['MAG_ZEROPOINT'] = '26.0' #### Run SExtractor on direct and alignment images ## direct image se.options['CATALOG_NAME'] = 'direct.cat' status = se.sextractImage(sci) threedhst.sex.sexcatRegions('direct.cat', 'direct.reg', format=2) directCat = threedhst.sex.mySexCat('direct.cat') #### Get the X/Y coords of the reference catalog #head = pyfits.getheader(sci, 0) #wcs = pywcs.WCS(head) if 'M0416' in sci: ra_list, dec_list, mag = np.loadtxt(os.getenv('HAWKI')+'/FrontierFields/HST/hlsp_frontier_subaru_suprimecam_macs0416-astrom_R_v1_cat.txt', unpack=True) if ('c4' in sci): ra_list, dec_list, mag = np.loadtxt(os.getenv('HAWKI')+'/FrontierFields/HST/M0416/macs0416_f814w_radec.cat', unpack=True) # if 'M0717' in sci: ra_list, dec_list, mag = np.loadtxt('subaru.radec', unpack=True) if ('M1149' in sci) | (field == 'M1149'): ra_list, dec_list, mag = np.loadtxt('/Users/brammer/Research/VLT/HAWKI/MACS1149/hlsp_frontier_subaru_suprimecam_macs1149-astrom_R_v1_cat.txt', unpack=True) if 'A2744' in sci: ra_list, dec_list, mag = np.loadtxt(os.getenv('HAWKI')+'/FrontierFields/HST/hlsp_frontier_subaru_suprimecam_abell2744-astrom_i_v1_cat.txt', unpack=True) if ('c1' in sci) | ('c4' in sci): ra_list, dec_list, mag = np.loadtxt(os.getenv('HAWKI')+'/FrontierFields/HST/abell2744_f814w_radec.cat', unpack=True) if align_data is not None: ra_list, dec_list, mag = align_data im = pyfits.open(sci) print sci sh = im[0].shape head = im[0].header head['CUNIT1'] = 'deg'; head['CUNIT2'] = 'deg' wcs = pywcs.WCS(head) x_image, y_image = wcs.wcs_sky2pix(ra_list, dec_list, 1) try: x_image, y_image = wcs.wcs_sky2pix(ra_list, dec_list, 1) except: x_image, y_image = wcs.wcs_world2pix(ra_list, dec_list, 1) ok = (x_image > 0) & (y_image > 0) & (x_image < sh[1]) & (y_image < sh[1]) x_image, y_image = x_image[ok], y_image[ok] fpr = open('align.reg','w') fpr.write('image\n') for i in range(ok.sum()): fpr.write('circle(%.6f, %.6f,0.3") # color=magenta\n' %(x_image[i], y_image[i])) fpr.close() # x_image, y_image = [], [] # # for ra, dec in zip(ra_list, dec_list): # x, y = wcs.wcs_sky2pix([[ra, dec]], 1)[0] # if (x > 0) & (y > 0) & (x < sh[1]) & (y < sh[1]): # x_image.append(x) # y_image.append(y) alignCat = catIO.EmptyCat() alignCat['X_IMAGE'] = np.array(x_image) alignCat['Y_IMAGE'] = np.array(y_image) xshift = 0 yshift = 0 rot = 0 scale = 1. xrms = 2 yrms = 2 NITER = 5 IT = 0 while (IT < NITER): IT = IT+1 #### Get x,y coordinates of detected objects ## direct image fp = open('direct.xy','w') for i in range(len(directCat.X_IMAGE)): fp.write('%s %s\n' %(directCat.X_IMAGE[i],directCat.Y_IMAGE[i])) fp.close() ## alignment image fp = open('align.xy','w') for i in range(len(alignCat.X_IMAGE)): fp.write('%s %s\n' %(np.float(alignCat.X_IMAGE[i])+xshift, np.float(alignCat.Y_IMAGE[i])+yshift)) fp.close() iraf.flpr() iraf.flpr() iraf.flpr() #### iraf.xyxymatch to find matches between the two catalogs pow = toler*1. try: os.remove('align.match') except: pass status1 = iraf.xyxymatch(input="direct.xy", reference="align.xy", output="align.match", tolerance=2**pow, separation=0, verbose=iraf.yes, Stdout=1) nmatch = 0 while status1[-1].startswith('0') | (nmatch < 10) | (float(status1[-3].split()[1]) > 40): pow+=1 os.remove('align.match') status1 = iraf.xyxymatch(input="direct.xy", reference="align.xy", output="align.match", tolerance=2**pow, separation=0, verbose=iraf.yes, Stdout=1) # nmatch = 0 for line in open('align.match').xreadlines( ): nmatch += 1 if verbose: for line in status1: print line #### Compute shifts with iraf.geomap iraf.flpr() iraf.flpr() iraf.flpr() try: os.remove("align.map") except: pass status2 = iraf.geomap(input="align.match", database="align.map", fitgeometry=fitgeometry, interactive=iraf.no, xmin=iraf.INDEF, xmax=iraf.INDEF, ymin=iraf.INDEF, ymax=iraf.INDEF, maxiter = 10, reject = 2.0, Stdout=1) if verbose: for line in status2: print line #fp = open(root+'.iraf.log','a') #fp.writelines(status1) #fp.writelines(status2) #fp.close() #### Parse geomap.output fp = open("align.map","r") for line in fp.readlines(): spl = line.split() if spl[0].startswith('xshift'): xshift += float(spl[1]) if spl[0].startswith('yshift'): yshift += float(spl[1]) if spl[0].startswith('xrotation'): rot = float(spl[1]) if spl[0].startswith('xmag'): scale = float(spl[1]) if spl[0].startswith('xrms'): xrms = float(spl[1]) if spl[0].startswith('yrms'): yrms = float(spl[1]) fp.close() #os.system('wc align.match') print 'Shift iteration #%d, xshift=%f, yshift=%f, rot=%f, scl=%f (rms: %5.2f,%5.2f)' %(IT, xshift, yshift, rot, scale, xrms, yrms) os.system('cat align.match | grep -v "\#" | grep [0-9] | awk \'{print "circle(", $1, ",", $2, ",4) # color=green"}\' > d.reg') os.system('cat align.match | grep -v "\#" | grep [0-9] | awk \'{print "circle(", $3, ",", $4, ",4) # color=magenta"}\' > a.reg') shutil.copy('align.map', sci.replace('.fits', '.align.map')) shutil.copy('align.match', sci.replace('.fits', '.align.match')) #### Cleanup if clean: rmfiles = ['align.cat', 'align.map','align.match','align.reg','align.xy', 'direct.cat','direct.reg','direct.xy'] for file in rmfiles: try: os.remove(file) except: pass fp = open(sci.replace('.fits', '.align.info'), 'w') fp.write('# image xshift yshift rot scale xrms yrms\n') fp.write('%s %.3f %.3f %.4f %.4f %.3f %.3f\n' %(sci, xshift, yshift, rot, scale, xrms, yrms)) if (np.abs(xshift) > shift_max) | (np.abs(yshift) > shift_max) | (xrms > rms_max) | (yrms > rms_max): print 'Shifts out of allowed range. Run again with increased shift_max to accept.' #return xshift, yshift, rot, scale, xrms, yrms ## Add a small shift that should come out easily with another ## shift iteration xshift, yshift, rot, scale, xrms, yrms = 2,2,0,1.0,-99,-99 for file in [sci, wht]: if ('r' in fitgeometry) & rot_only: xshift, yshift = 0, 0 #apply_offsets(file, [[xshift, yshift, rot, scale]]) from drizzlepac import updatehdr updatehdr.updatewcs_with_shift(file, sci, wcsname='DRZWCS', rot=rot,scale=scale, xsh=xshift, ysh=yshift, fit=None, xrms=xrms, yrms = yrms, verbose=False, force=True, sciext=0) if '_dr' in sci: im = pyfits.open(sci) h = im[0].header for i in range(h['NDRIZIM']): flt_str = h['D%03dDATA' %(i+1)] if 'sci,2' in flt_str: continue # flt_im = flt_str.split('[')[0] ext = int(flt_str.split('[')[1][:-1].split(',')[1]) updatehdr.updatewcs_with_shift(flt_im, sci, wcsname='GTWEAK', rot=rot, scale=scale, xsh=xshift, ysh=yshift, fit=None, xrms=xrms, yrms = yrms, verbose=False, force=True, sciext='SCI') # im = pyfits.open(file, mode='update') # wcs = pywcs.WCS(im[0].header) # wcs.rotateCD(-rot) # wcs.wcs.cd /= scale # # # im[0].header['CRPIX1'] += xshift # im[0].header['CRPIX2'] += yshift # # # for i in [0,1]: # for j in [0,1]: # im[0].header['CD%d_%d' %(i+1, j+1)] = wcs.wcs.cd[i,j] # # # im.flush() return xshift, yshift, rot, scale, xrms, yrms
def go(): os.chdir('/home/ec2-user/Mosaics') root = 'j123656p6215' root = 'j021732m0512' root = 'j141956p5255' root = 'j033236m2748' root = 'j100012p0210' # Sync needed files for GOODSN if root == 'j123656p6215': # os.system('aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/{0}_*dr*fits.gz" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" s3://grizli/Pipeline/ .'.format(root)) # os.system('aws s3 sync --exclude "*" --include "{0}*/Prep/*wcs.log" --include "{0}*/Prep/*shifts.*" s3://grizli/Pipeline/ .'.format(root)) os.system( 'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" s3://grizli/Pipeline/ .' .format(root)) else: #os.system('aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*_fl?.*" --include "{0}*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .'.format(root)) os.system( 'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .' .format(root)) #os.system('files=`find . |grep "dr[cz]_" |grep fits.gz |grep -v "\-ir_dr"`; for file in $files; do echo $file; gunzip -f $file; done') # Remake catalogs mos_files = glob.glob('{0}*/Prep/*_dr*sci.fits'.format(root)) mos_files.sort() thresh = 10 force = False for i, file in enumerate(mos_files): root = file.split('_drc_sci')[0].split('_drz_sci')[0] print(i, root) cat_file = root + '.cat.fits' if (not os.path.exists(cat_file)) | (force): prep.make_SEP_catalog(root=root, threshold=thresh, get_background=True, phot_apertures=[1 * u.arcsec]) os.chdir('Fine/') os.system('rsync -avz ../{0}*/Prep/*cat.fits .'.format(root)) os.system('ln -s ../{0}*/Prep/*_dr?_sci.fits .'.format(root)) os.system('rsync -avz ../{0}*/Prep/*visits.npy .'.format(root)) os.system('rsync -avz ../{0}*/Prep/*fail* .'.format(root)) visit_files = glob.glob('*visits.npy') #visit_files = glob.glob('*-tile-*visits.npy') visit_files.sort() filt = 'f850lp' all_visits = [] for f in visit_files: root_i = f.split('_visits.npy')[0] cat_file = root_i + '-' + filt + '.cat.fits' if not os.path.exists(cat_file): continue visits, _, _ = np.load(f, allow_pickle=True) has_failed = False for v in visits: has_failed |= os.path.exists(v['product'] + '.failed') if has_failed: continue all_visits.append({'product': root_i + '-' + filt}) # Fine alignment radec, ref_catalog = prep.get_radec_catalog(ra=189.2316435, dec=62.249739865958, product='xxx', reference_catalogs=['PS1'], radius=30) auto_script.fine_alignment(field_root='xxx', all_visits=all_visits, gaia_by_date=False, catalogs=['PS1'], redrizzle=False, shift_only=False, radec=radec, tol=1.e-3) ##### Done import numpy as np import os import glob from grizli import prep, utils from drizzlepac import updatehdr import astropy.io.fits as pyfits import astropy.wcs as pywcs fine_visits, fine_fit = np.load('xxx_fine.npy', allow_pickle=True) N = len(fine_visits) trans = np.reshape(fine_fit.x, (N, -1)) #/10. sh = trans.shape if sh[1] == 2: pscl = np.array([10., 10.]) trans = np.hstack([trans / pscl, np.zeros((N, 1)), np.ones((N, 1))]) elif sh[1] == 3: pscl = np.array([10., 10., 100]) trans = np.hstack([trans / pscl, np.ones((N, 1))]) elif sh[1] == 4: pscl = np.array([10., 10., 100, 100]) trans = trans / pscl # Update WCS # Update direct WCS for ix, direct in enumerate(fine_visits): root_i = direct['product'][:-7] print('\n\n\n\n##### ', ix, root_i, '#######\n\n\n') # Sync products os.system( 'aws s3 sync s3://grizli/Pipeline/{0}/Prep/ ./{0}/Prep/ --exclude "*" --include "{0}*drc_sci.fits.gz" --include "*_flc.*fits"' .format(root_i)) #os.system('aws s3 ls s3://grizli/Pipeline/{0}/Prep/'.format(root_i)) # Just visits and fail os.system( 'aws s3 sync s3://grizli/Pipeline/{0}/Prep/ ./{0}/Prep/ --exclude "*" --include "*visits.npy" --include "*fail*"' .format(root_i)) #direct = visits[ix] out_shift, out_rot = trans[ix, :2], trans[ix, 2] out_scale = trans[ix, 3] xyscale = trans[ix, :4] wcs_ref_file = str( '{0}/Prep/{0}-f850lp_drc_sci.fits.gz'.format(root_i)) wcs_ref = pywcs.WCS(pyfits.open(wcs_ref_file)[0].header, relax=True) for file in glob.glob('{0}/Prep/*flc.fits'.format(root_i)): prep.update_wcs_fits_log(file, wcs_ref, xyscale=xyscale, initialize=False, replace=('.fits', '.wcslog.fits'), wcsname='FINE') updatehdr.updatewcs_with_shift(file, wcs_ref_file, xsh=out_shift[0], ysh=out_shift[1], rot=out_rot, scale=out_scale, wcsname='FINE', force=True, reusename=True, verbose=True, sciext='SCI') ### Bug in astrodrizzle? Dies if the FLT files don't have MJD-OBS ### keywords im = pyfits.open(file, mode='update') im[0].header['MJD-OBS'] = im[0].header['EXPSTART'] im.flush() ## Fields with catalogs visit_files = glob.glob('j*/Prep/*f814w*visits.npy') visit_files.sort() ## Mosaic now that they're aligned # Shift parameters if False: os.system( 'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*cat.fits" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*shifts.*" --include "{0}*/Prep/*expflag*" s3://grizli-v1/Pipeline/ .' .format(root)) # No catalogs os.system( 'aws s3 sync --exclude "*" --include "{0}*/Prep/*expflag*" --include "{0}*/Prep/*fail*" --include "{0}*/Prep/*visits.npy" --include "{0}*/Prep/*[._]wcs*" --include "{0}*/Prep/*wcs.log" --include "{0}*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .' .format(root)) # ACS os.system( 'aws s3 sync --exclude "*" --include "{0}*acswfc*/Prep/*expflag*" --include "{0}*acswfc*/Prep/*fail*" --include "{0}*acswfc*/Prep/*visits.npy" --include "{0}*acswfc*/Prep/*[._]wcs*" --include "{0}*acswfc*/Prep/*shifts.*" s3://grizli-v1/Pipeline/ .' .format(root)) # Shifts os.system( 'echo "# root visit exp xsh ysh rot scl n xrms yrms" > shifts_results.log' ) os.system( 'grep "_fl" j*/Prep/*shifts.log | grep -v "\#" | sed "s/:/ /" | sed "s/.Prep./ /g" | sed "s/_shifts.log//" >> shifts_results.log' ) sh = utils.read_catalog('shifts_results.log') badsh = (sh['xrms'] > 1) | (sh['yrms'] > 1) #| (sh['n'] < 10) shift_skip_visits = list(np.unique(sh['visit'][badsh])) # Flag os.system('echo "group,x,expm,flag" > expflag_results.csv') os.system( 'grep "fits" j*/Prep/*expflag*txt | sed "s/:/,/g" | sed "s/.txt/ /" | sed "s/.Prep//" | sed "s/\//,/g" | sed "s/_raw/_flt/" >> expflag_results.csv' ) expf = utils.read_catalog('expflag_results.csv') bad_expf = expf['flag'] != 'NORMAL' exp_visits = np.unique(expf['group'][bad_expf]) exp_visits = [] # wcs os.system('echo "# dir visit x xs ys rot scl rms n" > shift_wcs.log') os.system( 'grep " 0 " j*/Prep/*wcs.log | sed "s/.Prep./ /" | sed "s/_wcs.log://" >> shift_wcs.log' ) shifts = utils.read_catalog('shift_wcs.log') skip = (shifts['n'] < 7) skip |= (shifts['xs'] == 0.) | (np.abs(shifts['rot']) > 0.2) skip_visits = [ '{0}_{1}'.format('_'.join(d.split('_')[:2]), v) for d, v in zip(shifts['dir'][skip], shifts['visit'][skip]) ] skip_visits += list(exp_visits) skip_visits += shift_skip_visits skip_visits = list(np.unique(skip_visits)) skip_keys = ['uds-12-bhm'] skip_keys += ['94s-245.0'] # bad shifts.txt skip_keys += ['sn2002zx-'] # bad scale visit_files = glob.glob('j*/Prep/*visits.npy') #visit_files = glob.glob('*shizels*/Prep/*visits.npy') all_visits = [] all_groups = [] all_info = [] bucket = 'grizli-v1' bucket = 'grizli-cosmos-v2' for file in visit_files: root_i = os.path.basename(file).split('_visits')[0] assoc = '_'.join(os.path.basename(file).split('_')[:2]) visits, groups, info = np.load(file) info['keep'] = False ic = 0 for v in visits: failed = glob.glob('{0}/Prep/{1}*fail*'.format( root_i, v['product'])) if len(failed) > 0: print(failed) continue for k in skip_keys: if k in v['product']: print('SKIP', v['product']) continue v['product'] = assoc + '_' + v['product'] if v['product'] in skip_visits: print('SKIP ', v['product']) continue v['awspath'] = ['{0}/Pipeline/{1}/Prep'.format(bucket, root_i) ] * len(v['files']) all_visits.append(v) for f in v['files']: info['keep'][info['FILE'] == f] = True all_info.append(info[info['keep'] == True]) for g in groups: failed = glob.glob('{0}/Prep/{1}*fail*'.format( root_i, g['direct']['product'])) if len(failed) > 0: print(failed) continue # for ext in ['direct', 'grism']: # g[ext]['product'] = assoc+'_'+g[ext]['product'] all_groups.append(g) import astropy.table all_info = astropy.table.vstack(all_info) if root == 'j123656p6215': out_root = 'gdn-' + root elif root == 'j021732m0512': out_root = 'uds-' + root elif root == 'j141956p5255': out_root = 'egs-' + root elif root == 'j033236m2748': out_root = 'gds-' + root elif root == 'j100012p0210': out_root = 'cos-' + root else: out_root = 'xxx-' + root np.save('{0}_visits.npy'.format(out_root), [all_visits, all_groups, all_info]) if True: os.system( 'aws s3 cp {0}_visits.npy s3://{1}/Mosaics/ --acl public-read'. format(out_root, bucket)) all_visits, all_groups, all_info = np.load( '{0}_visits.npy'.format(out_root)) ######## # Remake catalogs and drizzled images from drizzlepac.astrodrizzle import AstroDrizzle import numpy as np import glob import os from grizli import prep all_visits, all_groups, all_info = np.load( '{0}_visits.npy'.format(out_root)) filters = ['f814w', 'f140w', 'f160w', 'f105w', 'f098m', 'f850lp'] #filters = ['f850lp'] count, products = 0, [] for ii, direct in enumerate(all_visits[::-1]): filt = direct['product'].split('-')[-1] prod_files = glob.glob(direct['product'] + '*sci.fits') if (filt not in filters): continue else: products.append(direct['product']) if (len(prod_files) > 0): print('Skip ', direct['product']) continue else: count += 1 print('\n\n\n\n===========\n\n', ii, count, direct['product'], '\n\n\n========\n') #break isACS = '_flc' in direct['files'][0] if isACS: bits = 64 + 32 + 256 driz_cr_snr = '3.5 3.0' driz_cr_scale = '1.2 0.7' else: bits = 576 + 256 driz_cr_snr = '8.0 5.0' driz_cr_scale = '2.5 0.7' for aws, file in zip(direct['awspath'], direct['files']): if not os.path.exists(file): os.system('aws s3 cp s3://{0}/{1} ./'.format(aws, file)) cr_corr = False AstroDrizzle(direct['files'], output=direct['product'], clean=True, context=False, preserve=False, skysub=True, driz_separate=cr_corr, driz_sep_wcs=cr_corr, median=cr_corr, blot=cr_corr, driz_cr=cr_corr, driz_cr_corr=cr_corr, driz_cr_snr=driz_cr_snr, driz_cr_scale=driz_cr_scale, driz_combine=True, final_bits=bits, coeffs=True, resetbits=4096 * cr_corr, build=False, final_kernel='point', final_wht_type='IVM') # Remake catalog cat = prep.make_SEP_catalog(root=direct['product'], threshold=5) os.system('rm {0}*_[wbs]?[gt].fits'.format(direct['product'])) # Remove mosaic os.system('rm {0}*_sci.fits'.format(direct['product'])) # Remove FLC if isACS: for f in direct['files']: os.remove(f) # Make mosaics kwargs = auto_script.get_yml_parameters() mos_args = { 'mosaic_args': kwargs['mosaic_args'], 'fix_stars': kwargs['visit_prep_args']['fix_stars'], 'mask_spikes': kwargs['mask_spikes'], 'skip_single_optical_visits': kwargs['preprocess_args']['skip_single_optical_visits'] } #mos_args['mosaic_args']['ir_filters'] = ['F140W'] #mos_args['mosaic_args']['optical_filters'] = ['F814W'] mos_args['mosaic_args']['fill_mosaics'] = False mos_args['mosaic_args']['half_optical_pixscale'] = True mos_args['mosaic_args']['kernel'] = 'square' mos_args['mosaic_args']['pixfrac'] = 0.33 mos_args['mosaic_args']['wcs_params']['pixel_scale'] = 0.1 if root == 'j123656p6215': mos_args['mosaic_args']['kernel'] = 'point' mos_args['mosaic_args']['pixfrac'] = 0.33 mos_args['mosaic_args']['wcs_params']['pixel_scale'] = 0.1 #mos_args['mosaic_args']['wcs_params']['filters'] = ['F140W'] os.system('ln -s j*/Prep/*_fl?.fits .') auto_script.make_combined_mosaics(out_root, **mos_args)