def applyOffset(drzfile, fltfile, hlet=False): """Apply offset to an flt image from an aligned drizzled image""" tweakback.tweakback(drzfile, input=fltfile, origwcs='DRZWCS', verbose=True, force=True) if hlet: headerlet.write_headerlet(fltfile, 'HLFRED', output=None, sciext='SCI', wcsname='DRZWCS_2', wcskey='PRIMARY', destim=None, sipname=None, npolfile=None, d2imfile=None, author=None, descrip=None, history=None, nmatch=None, catalog=None, attach=True, clobber=False, logging=False)
def shift_wcs(im): try: im_wcs = wcs.WCS(fits.getheader(im, 1)) except: raise AssertionError('COULDN\'T FIND WCS FOR {}'.format(im)) if im_wcs.wcs.name == 'HSC': print '{} already aligned to HSC, skipping'.format(im) return f = [ln.strip('\n') for ln in open('hsc_shifts.txt').readlines()][-1] xsh, ysh, rot, scl, xrms, yrms = f.split()[1:] ref = 'hsc_shifts_wcs.fits' print im try: updatehdr.updatewcs_with_shift(im, ref, xsh=xsh, ysh=ysh, rot=rot, scale=scl, wcsname='HSC', force=False, sciext='SCI') tweakback.tweakback(im, force=False, origwcs='DRZWCS', newname='HSC', wcsname='HSC') except: print 'COULDN\'T UPDATE {}, ERRORING'.format(im) raise
def tback(drz): flts = tweakback.extract_input_filenames(drz) print 'Tweaking back exposures for {} with input ims:'.format(drz) for f in flts: print f try: tweakback.tweakback(drz) except: return
def tback(drz): """Runs tweakback to align flt/flcs with the drz/drc astrometry""" flts = tweakback.extract_input_filenames(drz) print 'Tweaking back exposures for {} with input ims:'.format(drz) for f in flts: print f try: tweakback.tweakback(drz) except: return
def shift_wcs(im): try: im_wcs = wcs.WCS(fits.getheader(im,1)) except: raise AssertionError('COULDN\'T FIND WCS FOR {}'.format(im)) if im_wcs.wcs.name == 'HSC': print '{} already aligned to HSC, skipping'.format(im) return f = [ln.strip('\n') for ln in open('hsc_shifts.txt').readlines()][-1] xsh, ysh, rot, scl, xrms, yrms = f.split()[1:] ref = 'hsc_shifts_wcs.fits' print im try: updatehdr.updatewcs_with_shift(im,ref,xsh=xsh,ysh=ysh,rot=rot,scale=scl,wcsname='HSC',force=False,sciext='SCI') tweakback.tweakback(im,force=False,origwcs='DRZWCS',newname='HSC',wcsname='HSC') except: print 'COULDN\'T UPDATE {}, ERRORING'.format(im) raise
def align_drizzled(images=['MACS2129-35-F814W_drc_sci.fits', 'MACS2129-36-F814W_drc_sci.fits']): from astropy.table import Table as table from drizzlepac import astrodrizzle, tweakreg, tweakback for image in images: root = image.split('_sci.fits')[0] se = threedhst.sex.SExtractor() se.options['WEIGHT_IMAGE'] = '%s_wht.fits[0]' %(root) se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT' # se.params['X_IMAGE'] = True; se.params['Y_IMAGE'] = True se.params['MAG_AUTO'] = True # se.options['CATALOG_NAME'] = '%s_sci.cat' %(root) se.options['FILTER'] = 'N' se.options['DETECT_THRESH'] = '5' se.options['ANALYSIS_THRESH'] = '5' # se.sextractImage('%s_sci.fits[0]' %(root)) threedhst.sex.sexcatRegions('%s_sci.cat' %(root), '%s_sci.reg' %(root), format=1) # t = table.read('%s_sci.cat' %(root), format='ascii.sextractor') np.savetxt('%s_sci.xy' %(root), np.array([t['X_IMAGE'], t['Y_IMAGE']]).T, fmt='%.7f') fp = open('%s_sci.catfile' %(root), 'w') fp.write('%s_sci.fits %s_sci.xy\n' %(root, root)) fp.close() reference = '%s_sci.xy' %(images[0].split('_sci.fits')[0]) for image in images[1:]: root = image.split('_sci.fits')[0] tweakreg.TweakReg(image, refimage=images[0], updatehdr=True, updatewcs=True, catfile='%s_sci.catfile' %(root), xcol=1, ycol=2, xyunits='pixels', refcat=reference, refxcol=1, refycol=2, refxyunits='pixels', shiftfile=False, searchrad=5, tolerance=12, wcsname='TWEAK3', interactive=False, residplot='No plot', see2dplot=False, clean=True, headerlet=True, clobber=True) tweakback.tweakback(image) pass
USE: python run_tweakback.py -drz drz.fits --> apply WCS solution from drz.fits to constituent flts (must be located in cwd). ''' __author__='D.M. HAMMER' __version__= 0.1 import os, glob, argparse, pyfits, pdb from pyraf import iraf import numpy as np from drizzlepac import tweakback from stsci.tools import teal if __name__=='__main__': # -- Parse input parameters parser = argparse.ArgumentParser(description='Run tweakback to apply WCS from drz to fl? images.') parser.add_argument('-drz', '--drzim',default='NONE', type=str, help='Input drizzled image (no wildcards). \ Default is NONE - requires input.') options = parser.parse_args() # -- initialize names of drizzled/flt(c) images if options.drzim == 'NONE': raise Exception('Must input a drizzled image.') drzim = options.drzim # -- run tweakback iraf.unlearn('tweakback') teal.unlearn('tweakback') tweakback.tweakback(drzim,verbose=True)
def drizzle_many(inputs_dict, output_dict={}, # darks_dict={}, kernel='lanczos3', reference_filter=None, updatewcs=False, clean=True, logfile=None, verbose=False, wcsname=None, exp_ratio=0.9, realign_only=False, # dark_med_factor=3, align_kwargs={}, drizzle_kwargs={}, realign_kwargs={}, tweakback_kwargs={}, redrizzle_kwargs={}): """ Drizzle together multiple sets of images to same aligned pixel scale Parameters ---------- inputs_dict : dictionary (Filter -> list of input files) output_dict : dictionary (Filter -> name of output file) darks_dict : dictionary (Filter -> name of dark file) """ logger = RegularPrint() if verbose else PrintRedirect(logfile) all_filters = list(inputs_dict.keys()) if reference_filter is None: reference_filter = all_filters[0] assert reference_filter in all_filters, ( f"{reference_filter} not in provided dictionary") # put reference filter first all_filters.remove(reference_filter) all_filters = [reference_filter] + list(all_filters) kwargs = { 'updatewcs': updatewcs, 'clean': clean, 'logfile': logfile, 'verbose': verbose} align_kwargs = {**kwargs, **align_kwargs} kwargs = { 'kernel': kernel, 'refimage': None, 'logfile': logfile, 'verbose': verbose, 'clean': clean} drizzle_kwargs = {**kwargs, **drizzle_kwargs} # Align and drizzle individual filters if not realign_only: print('*******************************') print('Initial Alignment and Drizzling') for filt in all_filters: print(f'--{filt}: Aligning') input_files = inputs_dict[filt] output_file = output_dict[filt] align_kwargs['wcsname'] = (f'INITIAL_{filt}') myalign(input_files, **align_kwargs) print(f'--{filt}: Drizzling') mydrizzle(input_files, output_file, **drizzle_kwargs) # Align drizzled images kwargs = { 'threshold': 200, 'nclip': 10, 'refimage': output_dict[reference_filter], 'wcsname': f'TWEAK_{reference_filter}', 'clean': clean, 'verbose': verbose, 'logfile': logfile, 'updatewcs': False, 'refimagefindcfg': {'threshold': 200}, } realign_kwargs = {**kwargs, **realign_kwargs} kwargs = { 'force': True, 'verbose': verbose } tweakback_kwargs = {**kwargs, **tweakback_kwargs} kwargs = { 'final_refimage': output_dict[reference_filter], 'final_wcs': True, 'final_rot': None, } kwargs = {**drizzle_kwargs, **kwargs} redrizzle_kwargs = {**redrizzle_kwargs, **kwargs} print('*****************************************************') print(f'Realign and Redrizzle relative to {reference_filter}') for filt in all_filters[1:]: input_files = inputs_dict[filt] output_file = output_dict[filt] print(f'--{filt}: Realigning') myalign(output_file, **realign_kwargs) print(f'--{filt}: TweakBack') with logger: print(tweakback_kwargs) tweakback.tweakback(output_file, input=input_files, **tweakback_kwargs) input_files = inputs_dict[filt] output_file = output_dict[filt] print(f'--{filt}: Redrizzling') mydrizzle(input_files, output_file, **redrizzle_kwargs) # dark_final = {filt: '{:s}_dark_drz.fits'.format(filt) for filt in all_filters} # # Create temorary dark files # print('--Creating temporary dark files') # dark_inputs = {} # dark_outputs = {} # with logger: # for filt in all_filters: # if filt in darks_dict: # dark_inputs[filt] = [] # for f in inputs_dict[filt]: # newfile = _copy_dark_header(f, darks_dict[filt]) # dark_inputs[filt].append(newfile) # dark_outputs[filt] = 'temp_dark_{:s}_drz.fits'.format(filt) # Drizzle the images # print('--Drizzling Images with {:s} kernel'.format(kernel)) # with logger: # # Drizzle first filter # first_filt = all_filters[0] # if len(inputs_dict[first_filt]) >= 4: # old_mode = d_kwgs.pop('combine_type', 'minmed') # d_kwgs['combine_type'] = 'median' # if len(inputs_dict[first_filt]) == 4: # old_nhigh = d_kwgs.pop('combine_nhigh', 0) # d_kwgs['combine_nhigh'] = 1 # astrodrizzle.AstroDrizzle( # inputs_dict[first_filt], output=output_dict[first_filt], **d_kwgs) # if len(inputs_dict[first_filt]) >= 4: # d_kwgs['combine_type'] = old_mode # if len(inputs_dict[first_filt]) == 4: # d_kwgs['combine_nhigh'] = old_nhigh # for filt in all_filters[1:]: # if len(inputs_dict[filt]) >= 4: # old_mode = d_kwgs.pop('combine_type', 'minmed') # d_kwgs['combine_type'] = 'median' # if len(inputs_dict[filt]) == 4: # old_nhigh = d_kwgs.pop('combine_nhigh', 0) # d_kwgs['combine_nhigh'] = 1 # # Align all other filters relative to first # astrodrizzle.AstroDrizzle( # inputs_dict[filt], output=output_dict[filt], # final_refimage=output_dict[first_filt], # **d_kwgs) # if len(inputs_dict[filt]) >= 4: # d_kwgs['combine_type'] = old_mode # if len(inputs_dict[filt]) == 4: # d_kwgs['combine_nhigh'] = old_nhigh # _add_flag(output_dict[filt], exp_ratio=exp_ratio) # Align and drizzle the dark images # dark_kwgs = d_kwgs.copy() # dark_kwgs.update( # {'driz_separate': False, # 'median': False, # 'blot': False, # 'driz_cr': False} # ) # for filt in dark_inputs: # if len(dark_inputs[filt]) < 3: # old_mode = d_kwgs.pop('combine_type', 'median') # d_kwgs['combine_type'] = 'minmed' # astrodrizzle.AstroDrizzle( # dark_inputs[filt], output=dark_outputs[filt], # final_refimage=output_dict[first_filt], **dark_kwgs) # if len(dark_inputs[filt]) < 3: # d_kwgs['combine_type'] = old_mode print('*************************') print('Accounting for Noise and Errors') # Compute and add dark values # dark_meds = {} # for filt in dark_outputs: # dark_meds[filt] = _add_dark(output_dict[filt], dark_outputs[filt], # dark_med_factor=dark_med_factor) # Add flag HDUs for filt in all_filters: _add_flag(output_dict[filt], exp_ratio=exp_ratio) # Compute Sky noise sky_vals = {} exposures = {} for filt in all_filters: with fits.open(output_dict[filt], mode='update') as h: sky_vals[filt] = Table(h['HDRTAB'].data).to_pandas()[['MDRIZSKY']].values.sum() / 2. exposures[filt] = h[0].header['EXPTIME'] h[0].header['SKYNOISE'] = (sky_vals[filt], 'Total sky value (sum of MDRIZSKY)') # h[0].header['NOISE'] = (sky_vals[filt] + dark_meds.get(filt, 0), # 'Total noise (SKYNOISE + DRKNOISE)') # Track later steps h[0].header['BKGDCOMP'] = ('PENDING', 'Has background been computed?') h[0].header['SEXTRACT'] = ('PENDING', 'Have SExtractor sources been masked?') h[0].header['MANUAL'] = ('PENDING', 'Have manual sources been masked?') h[0].header['REGIONS'] = ('PENDING', 'Have PCMD regions been computed?') for filt in all_filters: # noise = dark_meds.get(filt, 0.) + sky_vals[filt] noise = sky_vals[filt] exp = exposures[filt] print(f'--Sky Noise ({filt}): {noise:.1f} counts ({noise/exp:.3f} cps)') # print('--Sky: {:.1f} counts ({:.3f} cps)'.format(sky_vals[filt], sky_vals[filt]/exp)) # print('--Dark: {:.1f} counts ({:.3f} cps)'.format(dark_meds.get(filt, 0), dark_meds.get(filt, 0)/exp)) # if clean: # print('--Removing temporary files') # all_files = [] # for v in dark_inputs.values(): # if isinstance(v, list): # all_files.extend(v) # else: # all_files.append(v) # for v in dark_outputs.values(): # if isinstance(v, list): # all_files.extend(v) # else: # all_files.append(v) # for f in all_files: # try: # os.remove(f) # except FileNotFoundError: # pass print('--Done')
def offset_position(): """ Check grism extractions for 13580 program that put Vy2-2 just off of the left edge of the frame to test extrapolation of wavelength calibration. """ import astropy.io.fits as pyfits from astropy.table import Table as table import drizzlepac from drizzlepac import tweakreg, tweakback import stwcs import unicorn unicorn.candels.make_asn_files(uniquename=True) info = table.read('files.info', format='ascii.commented_header') for filter in ['F098M', 'F105W']: filter_files = list(info['FILE'][info['FILTER'] == filter]) # files = glob.glob('VY2-2*%s_asn.fits' %(filter)) for file in files: prep.prep_direct_grism_pair(direct_asn=file, grism_asn=False, radec='2mass.radec', scattered_light=False, skip_direct=False) # driz_images = glob.glob('VY2-2*%s_drz_sci.fits' %(filter)) tweakreg.TweakReg(driz_images, refimage=driz_images[0], updatehdr=True, updatewcs=True, catfile=None, xcol=2, ycol=3, xyunits='pixels', refcat=None, refxcol=1, refycol=2, refxyunits='degrees', shiftfile=True, outshifts='%s_shifts.txt' %(filter), outwcs='%s_wcs.fits' %(filter), searchrad=5, tolerance=12, wcsname='TWEAK', interactive=False, residplot='No plot', see2dplot=False, clean=True, headerlet=True, clobber=True) tweakback.tweakback(driz_images[1]) # drizzlepac.astrodrizzle.AstroDrizzle(filter_files, output='VY22-%s' %(filter), clean=True, skysub=False, final_scale=None, final_pixfrac=1, context=False, final_bits=576, preserve=False, driz_cr_snr='5.0 4.0', driz_cr_scale = '2.5 0.7') drizzlepac.astrodrizzle.AstroDrizzle(filter_files, output='VY22-%s' %(filter), clean=True, context=False, preserve=False, skysub=True, driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True) ### Put WCS from direct F105W images into G102 at same POS-TARG info = table.read('files.info', format='ascii.commented_header') idx = np.arange(len(info))[info['FILTER'] == 'F105W'] asn = threedhst.utils.ASNFile('../RAW/ibhj01030_asn.fits') for i in idx: direct = info['FILE'][i] dx, dy = info['POSTARG1'][i], info['POSTARG2'][i] ix_gris = (info['POSTARG1'] == dx) & (info['POSTARG2'] == dy) & (info['FILTER'] == 'G102') grism = info['FILE'][ix_gris][0] sign = {True:'+', False:'-'} # asn.product = 'VY22%s%02d%s%02d-F105W' %(sign[dx > 0], np.abs(dx), sign[dy > 0], np.abs(dy)) asn.exposures = [direct.split('_flt')[0]] asn.write(asn.product + '_asn.fits') # asn.product = 'VY22%s%02d%s%02d-G102' %(sign[dx > 0], np.abs(dx), sign[dy > 0], np.abs(dy)) asn.exposures = [grism.split('_flt')[0]] asn.write(asn.product + '_asn.fits') #### update WCS header imd = pyfits.open(direct) img = pyfits.open(grism) sci_ext=1 direct_WCS = stwcs.wcsutil.HSTWCS(imd, ext=sci_ext) drizzlepac.updatehdr.update_wcs(grism, sci_ext, direct_WCS, verbose=True) #### Make reference catalog root = 'VY22-F105W' se = threedhst.sex.SExtractor() se.aXeParams() se.copyConvFile() se.overwrite = True se.options['CHECKIMAGE_TYPE'] = 'SEGMENTATION, BACKGROUND' se.options['CHECKIMAGE_NAME'] = '%s_drz_seg.fits, %s_drz_bkg.fits' %(root, root) se.options['WEIGHT_TYPE'] = 'MAP_WEIGHT' se.options['WEIGHT_IMAGE'] = '%s_drz_wht.fits' %(root) se.options['WEIGHT_GAIN'] = 'Y' se.options['GAIN'] = '0' se.options['FILTER'] = 'Y' se.options['DETECT_THRESH'] = '2.' se.options['ANALYSIS_THRESH'] = '2.' se.options['DETECT_MINAREA'] = '10' se.options['MASK_TYPE'] = 'NONE' se.options['DEBLEND_NTHRESH'] = '64' se.options['DEBLEND_MINCONT'] = '0.1' se.options['SEEING_FWHM'] = '0.12' se.options['BACK_TYPE'] = 'MANUAL' se.options['BACKPHOTO_TYPE'] = 'LOCAL' se.options['MAG_ZEROPOINT'] = '%.2f' %(unicorn.reduce.ZPs['F105W']) se.options['CATALOG_TYPE'] = 'ASCII_HEAD' se.options['CATALOG_NAME'] = '%s_drz_sci.cat' %(root) status = se.sextractImage('%s_drz_sci.fits[0]' %(root)) threedhst.sex.sexcatRegions('%s_drz_sci.cat' %(root), '%s_drz_sci.reg' %(root), format=1) #### Make interlaced images files = glob.glob('VY22[+-]??[+-]??-F105W_asn.fits') for file in files: unicorn.reduce.interlace_combine(file.split('_asn')[0], growx=1, growy=1, NGROW=50, pad=60, view=False) unicorn.reduce.interlace_combine(file.split('_asn')[0].replace('F105W', 'G102'), growx=1, growy=1, NGROW=50, pad=60, view=False) red = unicorn.reduce red.adriz_blot_from_reference(pointing=file.split('_asn')[0], pad=60, NGROW=50, growx=1, growy=1, auto_offsets=False, ref_exp=0, ref_image='VY22-F105W_drz_sci.fits', ref_ext=0, ref_filter='F105W', seg_image='VY22-F105W_drz_seg.fits', cat_file='VY22-F105W_drz_sci.cat') ### extract spectra id = 798 files = glob.glob('VY22[+-]??[+-]??-F105W_asn.fits') for file in files: model = unicorn.reduce.GrismModel(root=file.split('-F10')[0], direct='F105W', grism='G102', growx=1, growy=1, grow_factor=1) model.twod_spectrum(id, miny=-30, refine=False, CONTAMINATING_MAGLIMIT=0) files = glob.glob('*2D.fits') yi, xi = np.indices((30,30)) xs, ys = np.zeros(len(files)), np.zeros(len(files)) xpix = xs*0 for i, file in enumerate(files): twod = unicorn.reduce.Interlace2D(file) xs[i] = np.sum(xi*twod.im['DSCI'].data/twod.im['DSCI'].data.sum()) ys[i] = np.sum(xi*twod.im['DSCI'].data/twod.im['DSCI'].data.sum()) xpix[i] = twod.im[0].header['X_PIX'] xs -= np.median(xs) #+ 0.5 ys -= np.median(ys) #xs -= 0.5 fig = plt.figure(figsize=[16,4]) fig.subplots_adjust(left=0.04, right=0.98, top=0.92) for i, file in enumerate(files): twod = unicorn.reduce.Interlace2D(file) w, f = twod.optimal_extract(twod.im['SCI'].data) c = {True: 'red', False: 'blue'} #plt.plot(w-np.diff(w)[0]*xs[i], f/twod.im['SENS'].data, alpha=0.5, marker='o', ms=2, label='%s, %s' %(file[4:7], file[7:10])) # , color=c['2-2' in file] ff = f*0. for k in range(ff.shape[0]): y0 = int(np.round(twod.im['YTRACE'].data[k])) ff[k] = np.sum(twod.im['SCI'].data[y0-4:y0+4, k]) # plt.plot(w-np.diff(w)[0]*xs[i], ff/twod.im['SENS'].data, alpha=0.5, marker='o', ms=2, label='%s, %s' %(file[4:7], file[7:10])) # , color=c['2-2' in file] #plt.plot(twod.oned.data['wave']-np.diff(w)[0]*xs[i], twod.oned.data['flux']/twod.oned.data['sensitivity'], alpha=0.5, marker='o', ms=2, label='%s, %s' %(file[4:7], file[7:10])) # , color=c['2-2' in file] # print file, np.diff(w)[0] #ds9.frame(i+1) #ds9.view(twod.im['DSCI'].data) PNe_lines = [9071.403457, 9534.921052, 10049.850283, 10833.000000, 12821.000000, 16112.000000, 16412.000000] for line in PNe_lines: plt.plot([line, line], [0.1,1.e5], color='black', linewidth=3, alpha=0.2, zorder=-5) #plt.plot(w-np.diff(w)[0]*xs[i]-np.diff(w)[0], f/twod.im['SENS'].data, alpha=0.5, color='green', marker='o', ms=2) plt.legend(loc='upper right', prop={'size':9}, title='POS-TARG') plt.title('VY2-2, G102, 13580') plt.xlim(8500, 11500) plt.ylim(700,14000) plt.ylim(600,64000) plt.semilogy() plt.xlabel(r'$\lambda$') plt.savefig('vy22-edge_v2.pdf') #, dpi=100) plt.close()
def hst_button( galaxies, skymethod='globalmin+match', instruments="ACS/WFC", prop_ids=None, filters=None, radius=None, filepath=None, download_data=True, correct_astrometry=True, create_mosaic=True, jy_conversion=True, verbose=False, log_filename='hst.log', ): """Create a HST mosaic, given a galaxy name. Using a galaxy name and radius, queries around that object, downloads available HST data and mosaics into a final product. It will create separate mosaics for each proposal ID, and the file structure will look like ``/galaxy/HST/proposal_id/galaxy_instrument_filter_proposal_id.fits``. N.B. I must confess to not being well-versed with HST data, so if anyone can help improve this please let me know. This data button uses a number of tools included in the drizzlepac Python package. This includes alignimages/tweakreg and astrodrizzle, which correct astrometry and are specifically tailored for the setup of HST data. This means that 1) creating mosaics with this will likely take a long time and 2) you will need a beefy computer (especially with regards to hard drive space). Args: galaxies (str or list): Names of galaxies to create mosaics for. Resolved by NED. skymethod (str, optional): Method used for AstroDrizzle's background matching step. In general, this can be left untouched but for mosaics with little overlap, it may be worth playing around with this. For instance, I've had some luck when there isn't much overlap between exposures using 'globalmin'. Options are 'localmin', 'globalmin+match', 'globalmin', and 'match'. Defaults to 'globalmin+match'. instruments (str or list, optional): Instrument to download data for. Can be any combination of 'ACS/WFC', 'WFC3/IR', 'WFC3/UVIS', 'WFPC2/PC', or 'WFPC2/WFC'. If you want all available data for all these instruments, select 'all', but this is not recommended! Defaults to 'ACS/WFC'. prop_ids (str or list, optional): Proposal IDs to download data for. Defaults to None, which will pull out all proposal IDs for each instrument. filters (str or list, optional): Filters to download data for. The script will look for each filter, for each instrument. Defaults to None, which will pull out all applicable filters for each instrument, for each proposal ID. radius (astropy.units.Quantity, optional): Radius around the galaxy to search for observations. Defaults to None, where it will query Ned to get size. filepath (str, optional): Path to save the working and output files to. If not specified, saves to current working directory. download_data (bool, optional): If True, will download data from MAST. Defaults to True. correct_astrometry (bool, optional): If True, will perform astrometric corrections to the downloaded data using alignimages. Defaults to True. create_mosaic (bool, optional): Switching this to True will mosaic the data using astrodrizzle as appropriate. Defaults to True. jy_conversion (bool, optional): Convert the mosaicked file from raw units to Jy/pix. Defaults to True. verbose (bool, optional): Can be used to suppress most of the output messages produced during the process. Mainly useful for debugging. Defaults to False. log_filename (str, optional): Will produce a stripped down log of what data the code is reducing. By default, will save to galaxy/hst.log. """ if isinstance(galaxies, str): galaxies = [galaxies] if isinstance(instruments,str): instruments = [instruments] if instruments == 'all': instruments = ['ACS/WFC', 'WFC3/IR','WFC3/UVIS', 'WFPC2/PC','WFPC2/WFC'] if isinstance(filters,str): filters = [filters] if isinstance(prop_ids,str): prop_ids = [prop_ids] if filepath is not None: os.chdir(filepath) orig_dir = os.getcwd() if radius is not None: original_radius = radius.copy() else: original_radius = None steps = [] if download_data: steps.append(1) if correct_astrometry: steps.append(2) if create_mosaic: steps.append(3) if jy_conversion: steps.append(4) # Set up folders for various corrections os.environ['CRDS_SERVER_URL'] = 'https://hst-crds.stsci.edu' os.environ['CRDS_PATH'] = orig_dir+'/reference_files' os.environ['iref'] = orig_dir+'/reference_files/references/hst/wfc3/' os.environ['jref'] = orig_dir+'/reference_files/references/hst/acs/' os.environ['uref'] = orig_dir+'/reference_files/references/hst/wfpc2/' # For large proposals, astrodrizzle can run into file open # issues so raise the max file open amount. _, hard = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE,(hard,hard)) # Change the temp directory -- if this gets filled up it can cause # problems. orig_tmpdir = os.environ['TMPDIR'] if not os.path.exists('tmp'): os.mkdir('tmp') os.environ['TMPDIR'] = orig_dir+'/tmp' for galaxy in galaxies: if not os.path.exists(galaxy): os.mkdir(galaxy) if not os.path.exists(galaxy+'/HST'): os.mkdir(galaxy+'/HST') if not verbose: # Various packages used here put out a lot of messages. Silence info messages. loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict] for logger in loggers: logger.setLevel(logging.ERROR) # Even if verbose is not True, still print out some useful messages to the # console. hst_logger = logging.getLogger('data_buttons') handler = logging.FileHandler(galaxy+'/'+log_filename,mode='w') hst_logger.addHandler(handler) hst_logger.addHandler(logging.StreamHandler()) hst_logger.setLevel(logging.INFO) hst_logger.info('Beginning '+galaxy) hst_logger.info(' ') hst_logger.info(' ') if radius is None: try: size_query = Ned.get_table(galaxy,table='diameters') radius = np.max(size_query['NED Major Axis'])/2*u.arcsec radius = radius.to(u.deg) except: hst_logger.warning(galaxy+' not resolved by Ned, using 0.2deg radius.') radius = 0.2*u.degree obs_table = Observations.query_criteria(objectname=galaxy, radius=radius, obs_type='all', obs_collection='HST') # Ignore any calibration observations. obs_table = obs_table[obs_table['intentType'] == 'science'] for instrument in instruments: # Pixel sizes for final mosaics selected to match the HLA. pix_size = {'ACS/HRC':0.025, 'ACS/SBC':0.03, 'ACS/WFC':0.05, 'NICMOS/NIC1':0.025, 'NICMOS/NIC2':0.05, 'NICMOS/NIC3':0.1, 'WFC3/IR':0.09, 'WFC3/UVIS':0.04, 'WFPC2/PC':0.05, 'WFPC2/WFC':0.1}[instrument] # Bits to consider good for drizzling. bits = {'ACS/HRC':256, 'ACS/SBC':256, 'ACS/WFC':256, 'NICMOS/NIC1':0, 'NICMOS/NIC2':0, 'NICMOS/NIC3':0, 'WFC3/IR':768, 'WFC3/UVIS':256, 'WFPC2/PC':'8,1024', 'WFPC2/WFC':'8,1024'}[instrument] # Filename extension, in order of preference. suffixes = {'ACS/WFC':['FLC','FLT'], 'WFC3/IR':['FLT'], 'WFC3/UVIS':['FLC','FLT'], 'WFPC2/PC':[['C0M','C1M']], 'WFPC2/WFC':[['C0M','C1M']], }[instrument] # The instruments often have / in the name, so account for # this in making folders and files. hst_logger.info('Beginning '+instrument) if not os.path.exists(galaxy+'/HST/'+instrument.replace('/','_')): os.mkdir(galaxy+'/HST/'+instrument.replace('/','_')) reset_filters = False instrument_table = obs_table[obs_table['instrument_name'] == instrument] reset_prop_ids = False if not prop_ids: prop_ids = list(np.unique(instrument_table['proposal_id'])) reset_prop_ids = True hst_logger.info('Available proposal IDs: '+','.join(prop_ids)) hst_logger.info(' ') for prop_id in prop_ids: hst_logger.info('Proposal ID: '+str(prop_id)) prop_table = instrument_table[instrument_table['proposal_id'] == prop_id] if not filters: filters = list(np.unique(prop_table['filters'])) reset_filters = True hst_logger.info('Available filters: '+','.join(filters)) for hst_filter in filters: # If we have a highly illegal filter, just skip. # TODO: This needs to be sorted for some fringe # cases, probably. if not hst_filter[0] == 'F': continue hst_logger.info('Filter: '+str(hst_filter)) # Pull out available data and download. filter_table = prop_table[prop_table['filters'] == hst_filter] if len(filter_table) == 0: hst_logger.warning('No available data to download. Skipping...') continue data_products_id = Observations.get_product_list(filter_table) for suffix in suffixes: download_table = Observations.filter_products(data_products_id, productSubGroupDescription=suffix, mrp_only=False) if len(download_table) > 0: break if isinstance(suffix,list): filename_exts = [ext.lower() for ext in suffix] else: filename_exts = [suffix.lower()] hst_logger.info(instrument+'/'+prop_id+'/'+hst_filter) if not os.path.exists(galaxy+ '/HST/'+ instrument.replace('/','_')+ '/'+ hst_filter): os.mkdir(galaxy+ '/HST/'+ instrument.replace('/','_')+ '/'+ hst_filter) if not os.path.exists(galaxy+'/HST/'+prop_id): os.mkdir(galaxy+'/HST/'+prop_id) full_filepath = (galaxy+ '/HST/'+ instrument.replace('/','_')+ '/'+ hst_filter+ '/' +prop_id) if not os.path.exists(full_filepath): os.mkdir(full_filepath) if 1 in steps: # Download files download_mast(download_table, download_dir="hst_temp/" + galaxy) if not os.path.exists(full_filepath+'/raw'): os.mkdir(full_filepath+'/raw') if not os.path.exists(full_filepath+'/outputs'): os.mkdir(full_filepath+'/outputs') # Pull out the relevant files, and move to base folder. for filename_ext in filename_exts: matches = [] for root, _, filenames in os.walk("hst_temp/" + galaxy): for filename in fnmatch.filter( filenames, "*_"+filename_ext+".fits" ): matches.append(os.path.join(root, filename)) for match in matches: filename = match.split('/') os.rename(match,full_filepath+'/raw/'+filename[-1]) # Clean up any temporary files. shutil.rmtree("hst_temp/" + galaxy, ignore_errors=True) filename_ext = filename_exts[0] hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits') if 2 in steps: # First, update the WCS information in case it's # required. for filename_ext in filename_exts: hst_files = glob.glob(full_filepath+'/raw/*_'+filename_ext+'.fits') crds.assign_bestrefs(hst_files, sync_references=True) # For WFPC2, the CRDS doesn't download everything # needed. Download the GEIS data files and # rerun the bestrefs assignment. if 'WFPC2' in instrument: geis_hdrs = glob.glob(os.environ['uref']+'/*h') for geis_hdr in geis_hdrs: geis_data = geis_hdr[:-1]+'d' if not os.path.exists(geis_data): geis_data = geis_data.split('/')[-1] print(geis_data) print(os.environ['uref']) wget.download( os.environ['CRDS_SERVER_URL']+'/unchecked_get/references/hst/'+geis_data, out=os.environ['uref']) crds.assign_bestrefs(hst_files,sync_references=True) for hst_file in hst_files: stwcs.updatewcs.updatewcs(hst_file, use_db=False) os.chdir(full_filepath+'/raw') filename_ext = filename_exts[0] hst_files = glob.glob('*_'+filename_ext+'.fits') # Normalize all files. photeq.photeq(', '.join(hst_files),readonly=False) os.rename('photeq.log','../outputs/photeq.log') if 'WFPC' in instrument: # Using tweakreg, align each frame to GAIA. gaia_table = Gaia.query_object_async(coordinate=galaxy, radius=2*radius) ras = gaia_table['ra'] decs = gaia_table['dec'] source_table = Table([ras,decs]) source_table.write('gaia.cat', format='ascii.fast_commented_header') tweakreg.TweakReg(hst_files, imagefindcfg={'threshold':5,'conv_width':3}, refcat='gaia.cat', #expand_refcat=True, enforce_user_order=False, shiftfile=True, outshifts='shifts.txt', searchrad=10, minobj=5, separation=0, updatehdr=True, reusename=True, wcsname='TWEAK', interactive=False, fitgeometry='general', clean=True, see2dplot=False ) # Update the c1m files to use the TWEAK # wcs for hst_file in hst_files: dq_file = hst_file.replace('c0','c1') tweakback.tweakback(hst_file, dq_file, newname='TWEAK') plot_files = glob.glob('*.png') for plot_file in plot_files: os.remove(plot_file) cat_files = glob.glob('*.coo') for cat_file in cat_files: os.remove(cat_file) os.rename('shifts_wcs.fits','../outputs/shifts_wcs.fits') os.rename('tweakreg.log','../outputs/tweakreg.log') os.rename('shifts.txt','../outputs/shifts.txt') elif 'ACS' in instrument or 'WFC3' in instrument: # Correct astrometry using alignimages. First, # correct each frame separately. pool = mp.Pool(mp.cpu_count()) suitable_hst_files = pool.map(astrometric_correction, hst_files) pool.close() suitable_hst_files = [x for x in suitable_hst_files if x is not None] if len(suitable_hst_files) == 0: hst_logger.warning('Failure with astrometry corrections. Skipping') os.chdir(orig_dir) continue # Now, align every suitable frame simultaneously. output_table = astrometric_correction(suitable_hst_files) with open('../outputs/astrometry.pkl','wb') as table_file: pickle.dump(output_table,table_file) else: raise Exception('Unknown instrument!') os.chdir(orig_dir) os.chdir(full_filepath) if 3 in steps: os.chdir('raw') if 'WFPC2' in instrument: hst_files = glob.glob('*_c0m.fits') wcskey = 'TWEAK' elif 'ACS' in instrument or 'WFC3' in instrument: with open('../outputs/astrometry.pkl','rb') as table_file: output_table = pickle.load(table_file) # We only want fits where an acceptable astrometric # solution has been found. suitable_fits = np.where(output_table['fit_qual'] < 5) # (output_table['fit_qual'] >= 1) hst_files = list(output_table[suitable_fits]['imageName']) if len(output_table[suitable_fits]) == 0: hst_logger.warning('Failure with astrometry corrections. Skipping') os.chdir(orig_dir) continue wcskey = ' ' else: raise Exception('Unknown instrument!') # Following Dalcanton+ (2012), group exposures into # long (>50s) and short (<=50s), and process for cosmic # rays separately exp_times = [] for hst_file in hst_files: hdu = fits.open(hst_file)[0] exp_time = hdu.header['EXPTIME'] exp_times.append(exp_time) for exp_group in ['short','long']: hst_files_group = [] for i in range(len(exp_times)): if exp_times[i] > 50 and exp_group == 'long': hst_files_group.append(hst_files[i]) elif exp_times[i] <= 50 and exp_group == 'short': hst_files_group.append(hst_files[i]) if len(hst_files_group) == len(hst_files): exp_group = '' if len(hst_files_group) == 0: continue if len(exp_group) > 0: output_name = '../outputs/'+galaxy+'_'+exp_group drizzle_log_name = '../outputs/astrodrizzle_'+exp_group+'.log' else: output_name = '../outputs/'+galaxy drizzle_log_name = '../outputs/astrodrizzle.log' # Perform the mosaicking. Generally, use iminmed. # However, sometimes iminmed will fail so # for the other instruments we'll use imedian as # a fallback. combine_types = ['iminmed','imedian'] if 'WFPC2' in instrument: combine_nhigh = 1 else: combine_nhigh = 0 for combine_type in combine_types: try: astrodrizzle.AstroDrizzle( input=hst_files_group, output=output_name, preserve=False, clean=True, combine_type=combine_type, combine_nhigh=combine_nhigh, skymethod=skymethod, sky_bits=bits, driz_sep_bits=bits, driz_sep_fillval=99999, combine_hthresh=90000, final_scale=pix_size, final_bits=bits, final_fillval=0, wcskey=wcskey, final_rot=0, ) break except ValueError: pass # Move the AstroDrizzle log. os.rename('astrodrizzle.log', drizzle_log_name) # Move back to the original directory. os.chdir(orig_dir) if 4 in steps: mosaic_outputs = glob.glob(full_filepath+'/outputs/*_sci.fits') for mosaic_output in mosaic_outputs: # Replace any fillvals with NaNs. hdu = fits.open(mosaic_output)[0] hdu.data[hdu.data == 0] = np.nan fits.writeto(mosaic_output, hdu.data,hdu.header, overwrite=True) if '_long_' in mosaic_output.split('/')[-1]: new_filename = (galaxy+ '/HST/' +prop_id +'/' +galaxy +'_' +instrument.replace('/','_') +'_' +hst_filter +'_' +prop_id +'_long.fits') elif '_short_' in mosaic_output.split('/')[-1]: new_filename = (galaxy+ '/HST/' +prop_id +'/' +galaxy +'_' +instrument.replace('/','_') +'_' +hst_filter +'_' +prop_id +'_short.fits') else: new_filename = (galaxy+ '/HST/' +prop_id +'/' +galaxy +'_' +instrument.replace('/','_') +'_' +hst_filter +'_' +prop_id +'.fits') convert_to_jy(mosaic_output, new_filename) if reset_filters: filters = None hst_logger.info(' ') if reset_prop_ids: prop_ids = None hst_logger.info(' ') if original_radius is None: radius = None else: radius = original_radius.copy() # Clear out the tmp folder and reset to the original. shutil.rmtree('tmp/', ignore_errors=True) os.environ['TMPDIR'] = orig_tmpdir
def runpipe( outroot, onlyfilters=[], onlyepochs=[], # Run all the processing steps doall=False, # Setup : copy flts into sub-directories dosetup=False, epochlistfile=None, # construct a ref image dorefim=False, # Single Visit tweakreg/drizzle pass : dodriz1=False, drizcr=False, intravisitreg=False, # Register to a given image or epoch, visit and filter doreg=False, refim=None, refepoch=None, refvisit=None, reffilter=None, # Drizzle registered flts by epoch and filter dodriz2=False, # make diff images dodiff=False, tempepoch=0, refcat=None, interactive=False, threshold=4, peakmin=None, peakmax=None, rfluxmax=27, rfluxmin=14, searchrad=1.5, mjdmin=0, mjdmax=0, epochspan=5, ra=None, dec=None, rot=0, imsize_arcsec=None, pixscale=None, pixfrac=None, wht_type='ERR', clobber=False, verbose=True, debug=False ): """ Primary pipeline function. Executes all the intermediate steps: register, sort, drizzle, subtract, mask """ import pyfits import shutil if debug : import pdb; pdb.set_trace() from drizzlepac.tweakback import tweakback topdir = os.path.abspath( '.' ) fltdir = outroot + '.flt' if doall : dosetup=True dorefim=True dodriz1=True doreg=True dodriz2=True dodiff=True if onlyfilters : if type(onlyfilters)==str : onlyfilters = onlyfilters.lower().split(',') onlyfilters = [ filt[:5].lower() for filt in onlyfilters ] if type(onlyepochs) in [str,int,float] : onlyepochs = [ int(ep) for ep in str(onlyepochs).split(',') ] # STAGE 0 : (always runs) # get a list of exposures and epochs, sorting flt files into epochs fltlist = glob.glob( "%s/*fl?.fits"%fltdir ) if not len( fltlist ) : raise( exceptions.RuntimeError( "There are no flt/flc files in %s !!"%fltdir) ) explist = exposures.get_explist( fltlist, outroot=outroot ) if not epochlistfile : epochlistfile = "%s_epochs.txt"%explist[0].outroot if os.path.exists( epochlistfile ) and not clobber : print( "%s exists. Adopting existing epoch sorting."%epochlistfile ) exposures.read_epochs( explist, epochlistfile, checkradec=[ra,dec], onlyfilters=onlyfilters ) else : exposures.define_epochs( explist, epochspan=epochspan, mjdmin=mjdmin, mjdmax=mjdmax ) exposures.print_epochs( explist, outfile=epochlistfile, verbose=verbose, clobber=clobber, checkradec=[ra,dec], onlyfilters=onlyfilters, onlyepochs=onlyepochs ) if refim and not os.path.exists( refim ) : raise exceptions.RuntimeError( 'Ref image %s does not exist.'%refim ) if not refim : # No refimage has been specified, so set the default refimage name refdrzdir = '%s.refim'%outroot refimbasename = '%s_wcsref_sci.fits'%(outroot) refim = os.path.abspath( os.path.join( refdrzdir, refimbasename ) ) new_explist = [] for exp in explist: if onlyfilters and exp.filter not in onlyfilters : continue elif onlyepochs and exp.epoch not in onlyepochs : continue else: new_explist.append( exp ) explist = new_explist # STAGE 1 : # copy pristine flt files into epoch sub-directories if dosetup : if verbose : print("SNDRIZZLE : (1) SETUP : copying flt files into subdirs") exposures.copy_to_epochdirs( explist, checkradec=[ra,dec], onlyfilters=onlyfilters, onlyepochs=onlyepochs, verbose=verbose, clobber=clobber ) FEVgrouplist = sorted( np.unique( [ exp.FEVgroup for exp in explist ] ) ) FEgrouplist = sorted( np.unique( [ exp.FEgroup for exp in explist ] ) ) filterlist = sorted( np.unique( [ exp.filter for exp in explist ] ) ) epochlist = sorted( np.unique([ exp.epoch for exp in explist ] ) ) # STAGE 2 : # Construct the WCS reference image if dorefim : if verbose : print("SNDRIZZLE : (2) REFIM : Constructing WCS ref image.") if os.path.exists( refim ) and clobber : os.remove( refim ) if os.path.exists( refim ) : print("%s already exists. Not clobbering."%refim) else : refdrzdir = os.path.dirname( refim ) if verbose : print( " Constructing reference image %s in %s"%( os.path.basename(refim), refdrzdir ) ) # Collect the necessary flt files for constructing the ref image if not refepoch : refepoch = np.min( epochlist ) if not reffilter : reffilter = sorted( [ exp.filter for exp in explist if exp.epoch==refepoch ] )[0] reffilter = reffilter.lower() # if refvisit is not set, then find the maximum depth visit and use that if not refvisit : visits, exp_times, visit_depth = np.array( ([], [], []) ) for exp in explist: if (exp.epoch != refepoch) and (exp.filter != reffilter): continue visits = np.append( visits, exp.visit ) exp_times = np.append( exp_times, exp.exposure_time ) unique_visits = np.unique( visits ) for u in unique_visits: ind = np.where( visits == u ) visit_depth = np.append( visit_depth, np.sum(exp_times[ind]) ) max_ind = np.argmax( visit_depth ) refvisit = unique_visits[max_ind] refvisit = refvisit.upper() explistRI = sorted( [ exp for exp in explist if exp.epoch==refepoch and exp.filter==reffilter and exp.visit==refvisit ] ) refdrzdir = os.path.dirname( refim ) if not os.path.isdir( refdrzdir ) : os.makedirs( refdrzdir ) for exp in explistRI : fltfile = os.path.basename( exp.filename ) refsrcdir = os.path.abspath( exp.epochdir ) shutil.copy( os.path.join( refsrcdir, fltfile ), refdrzdir ) fltlistRI = [ exp.filename for exp in explistRI ] refimroot = '%s_wcsref'%outroot os.chdir( refdrzdir ) refimsci, refimwht = drizzle.secondDrizzle( fltlistRI, refimroot, refimage=None, ra=ra, dec=dec, rot=rot, imsize_arcsec=imsize_arcsec, wht_type=wht_type, pixscale=pixscale, pixfrac=pixfrac, clobber=clobber, verbose=verbose, debug=debug ) os.rename(refimsci,refim) os.chdir(topdir) print refvisit # STAGE 3 : # Drizzle together each drizzle group (same epoch, visit and # filter), using almost-default parameters, doing CR rejection # and applying intravisit registrations if requested. The output # is a drz_sci.fits file in the native rotation. if dodriz1 : if verbose : print("SNDRIZZLE : (3) DRIZ1 : first astrodrizzle pass.") for FEVgroup in FEVgrouplist : explistFEV = [ exp for exp in explist if exp.FEVgroup == FEVgroup ] thisepoch = explistFEV[0].epoch thisfilter = explistFEV[0].filter if onlyepochs and thisepoch not in onlyepochs : continue if onlyfilters and thisfilter not in onlyfilters : continue epochdir = explistFEV[0].epochdir fltlistFEV = [ exp.filename for exp in explistFEV ] outrootFEV = '%s_%s_nat'%(outroot,FEVgroup) os.chdir( epochdir ) outsciFEV = '%s_%s_sci.fits'%(outrootFEV,explistFEV[0].drzsuffix) if os.path.exists( outsciFEV ) and clobber : os.remove( outsciFEV ) if os.path.exists( outsciFEV ) : if verbose: print("%s exists. Not clobbering."%outsciFEV ) os.chdir( topdir ) continue if intravisitreg : # run tweakreg for intravisit registration tweaks register.intraVisit( fltlistFEV, peakmin=peakmin, peakmax=peakmax, threshold=threshold, interactive=interactive, debug=debug ) drizzle.firstDrizzle( fltlistFEV, outrootFEV, driz_cr=drizcr, wcskey=((intravisitreg and 'INTRAVIS') or '') ) # TODO : Update the WCS of the refim so that it matches the reference catalog #if refcat : # if verbose : print( " Registering reference image %s to ref catalog %s"%(refim,refcat)) # register.toCatalog( refim, refcat, refim, rfluxmax=rfluxmax, rfluxmin=rfluxmin, # searchrad=searchrad, peakmin=peakmin, peakmax=peakmax, threshold=threshold, # interactive=interactive, debug=debug ) os.chdir( topdir ) if doreg or dodriz2 : if not os.path.exists( refim ): raise exceptions.RuntimeError("No refim file %s! Maybe you should re-run with dorefim=True."%refim) else : refim = os.path.abspath( refim ) # Fix the output ra and dec center point if not provided by the user. if not ra and not dec : ra = pyfits.getval( refim, "CRVAL1" ) dec = pyfits.getval( refim, "CRVAL2" ) # STAGE 4 : # Run tweakreg to register all the single-visit drz images # to a common WCS defined by the refcat/refim, updating the drz file headers. # Then use tweakback to propagate that back into the flt files if doreg : if verbose : print("SNDRIZZLE : (4) REG : running tweakreg.") for FEVgroup in FEVgrouplist : explistFEV = [ exp for exp in explist if exp.FEVgroup == FEVgroup ] thisepoch = explistFEV[0].epoch thisfilter = explistFEV[0].filter if onlyepochs and thisepoch not in onlyepochs : continue if onlyfilters and thisfilter not in onlyfilters : continue epochdir = explistFEV[0].epochdir fltlistFEV = [ exp.filename for exp in explistFEV ] outrootFEV = '%s_%s_nat'%(outroot,FEVgroup) outsciFEV = '%s_%s_sci.fits'%(outrootFEV,explistFEV[0].drzsuffix) refimpath = os.path.abspath(refim) if refcat : refcatpath = os.path.abspath(refcat) else : refcatpath = None os.chdir( epochdir ) if not os.path.exists( outsciFEV ) : exceptions.RuntimeError( "Missing %s."%outsciFEV ) # register to the ref image and ref catalog origwcs = pyfits.getval( outsciFEV,'WCSNAME').strip() wcsname = register.toRefim( outsciFEV, refim=refimpath, refcat=refcatpath, searchrad=searchrad, peakmin=peakmin, peakmax=peakmax, threshold=threshold, interactive=interactive, clobber=clobber, debug=debug ) # Run tweakback to update the constituent flts tweakback( outsciFEV, input=fltlistFEV, origwcs=origwcs, wcsname=wcsname, verbose=verbose, force=clobber ) os.chdir(topdir) # STAGE 5 # Second and final astrodrizzle pass, wherein we rerun # astrodrizzle to get wcs- and pixel-registered drz images # combining all flt files with the same filter and epoch. if dodriz2 : if verbose : print("SNDRIZZLE : (5) DRIZ2 : second astrodrizzle pass.") for FEgroup in FEgrouplist : explistFE = [ exp for exp in explist if exp.FEgroup == FEgroup ] thisepoch = explistFE[0].epoch thisfilter = explistFE[0].filter if onlyepochs and thisepoch not in onlyepochs : continue if onlyfilters and thisfilter not in onlyfilters : continue epochdir = explistFE[0].epochdir fltlistFE = [ exp.filename for exp in explistFE ] outrootFE = '%s_%s_reg'%(outroot,FEgroup) err = None os.chdir( epochdir ) outsciFE = '%s_%s_sci.fits'%(outrootFE,explistFE[0].drzsuffix) if os.path.exists( outsciFE ) and clobber : os.remove( outsciFE ) if os.path.exists( outsciFE ) : if verbose: print("%s exists. Not clobbering."%outsciFE ) os.chdir( topdir ) continue outsciFE, outwhtFE = drizzle.secondDrizzle( fltlistFE, outrootFE, refimage=refim, ra=ra, dec=dec, rot=rot, imsize_arcsec=imsize_arcsec, wht_type=wht_type, pixscale=pixscale, pixfrac=pixfrac, clobber=clobber, verbose=verbose, debug=debug ) outbpxFE = outwhtFE.replace('_wht','_bpx') outbpxFE = badpix.zerowht2badpix( outwhtFE, outbpxFE, verbose=verbose, clobber=clobber ) os.chdir( topdir ) # STAGE 6 # Define a template epoch for each filter and subtract it from the other epochs if dodiff : if verbose : print("SNDRIZZLE : (6) DIFF : subtracting template images.") for filter in filterlist : if onlyfilters and filter not in onlyfilters : continue template = None if tempepoch == None : # User did not define a template epoch, so we default # to use the first available epoch for epoch in epochlist : explistFE = [ exp for exp in explist if exp.filter==filter and exp.epoch==epoch ] if len(explistFE)==0: continue drzsuffix = explistFE[0].drzsuffix epochdir = explistFE[0].epochdir FEgroup = explistFE[0].FEgroup outrootFE = '%s_%s'%(outroot, FEgroup ) if os.path.isfile( os.path.join( epochdir, outrootFE+"_reg_%s_sci.fits"%drzsuffix )) : tempepoch = epoch break else : explistF = [ exp for exp in explist if exp.filter==filter ] drzsuffix = explistF[0].drzsuffix # now do the subtractions tempdir = outroot+'.e%02i'%tempepoch tempsci = os.path.join( os.path.abspath(tempdir), '%s_%s_e%02i_reg_%s_sci.fits'%(outroot,filter,tempepoch,drzsuffix)) tempwht = tempsci.replace('sci.fits','wht.fits') tempbpx = tempsci.replace('sci.fits','bpx.fits') topdir = os.path.abspath( '.' ) for epoch in epochlist : if onlyepochs and epoch not in onlyepochs : continue if epoch == tempepoch : continue explistFE = [ exp for exp in explist if exp.filter==filter and exp.epoch==epoch ] if len(explistFE)==0 : continue epochdirFE = explistFE[0].epochdir outrootFE = "%s_%s"%(outroot,explistFE[0].FEgroup) drzsuffix = explistFE[0].drzsuffix thisregsci = "%s_reg_%s_sci.fits"%(outrootFE,drzsuffix) thisregwht = "%s_reg_%s_wht.fits"%(outrootFE,drzsuffix) thisregbpx = "%s_reg_%s_bpx.fits"%(outrootFE,drzsuffix) os.chdir( epochdirFE ) if not os.path.isfile( thisregsci ) : os.chdir( topdir ) continue thisdiffim = outrootFE + "-e%02i_sub_sci.fits"%tempepoch if not os.path.isfile( tempsci ) or not os.path.isfile( thisregsci) : print( "Can't create diff image %s"%thisdiffim ) print( " missing either %s"%tempsci ) print( " or %s"%thisregsci ) os.chdir( topdir ) continue diffim = imarith.imsubtract( tempsci, thisregsci, outfile=thisdiffim, clobber=clobber, verbose=verbose, debug=debug) diffwht = badpix.combine_ivm_maps( thisregwht, tempwht, diffim.replace('sci.fits','wht.fits'), clobber=clobber, verbose=verbose ) diffbpx = badpix.unionmask( tempbpx, thisregbpx, diffim.replace('sci.fits','bpx.fits'), clobber=clobber, verbose=verbose) diffim_masked = badpix.applymask( diffim, diffbpx, clobber=clobber, verbose=verbose) print("Created diff image %s, wht map %s, and bpx mask %s"%( diffim_masked, diffwht, diffbpx ) ) os.chdir( topdir ) return 0
def drizzle_fields( obsDates, fltList, cluster, filter, search_rad=1, thresh=1): ''' PURPOSE : IF THERE ARE OBSERVATIONS FROM DIFFERENT DATES / OBSRUNS THEN DRIZZLE THE DATES THAT ARE THE SAME TOGETHER THEN TWEAK THEM TO ONE REFERENCE DRIZZLE IMAGE TWEAK BACK TO THE FLT FITS INPUTS : OBSDATES : CAN BE ANY UNIQUE IDENTIFIER TO NAME OF THE RUN OR DATE thresh=1.0 search_rad=1.0 #Often this parameter is too big if the images hardly overlap. Keep at 1 if limited sources ''' outputfilename = cluster+'_'+filter uniqueObs = np.unique(np.array(obsDates)) print 'MORE THAN ONE OBSERVATION DATE/RUN SO DRIZZLING INDIVIDUALLY' print uniqueObs call(["mkdir", "keep"]) for iDate in uniqueObs: obsDrizzle = np.array(fltList)[ np.array(obsDates) == str(iDate) ] drizzleString = ",".join(obsDrizzle) print iDate print obsDrizzle if not os.path.isfile("keep/"+str(iDate)+"_drz_sci.fits"): astrodrizzle.AstroDrizzle(drizzleString, \ output=str(iDate), \ final_wcs=True, \ final_scale=0.03, \ final_pixfrac=0.8, \ combine_type='iminmed') call(["cp",str(iDate)+"_drz_sci.fits","keep"]) else: call(["cp","keep/"+str(iDate)+"_drz_sci.fits","."]) #3. NOW TWEAK EACH DRIZZLED IMAGE #----------------------------------------- #First sextract them using tweak_sextract drzList=[] for iDate in uniqueObs: drzList.append(str(iDate)+"_drz_sci.fits") try: drzList.remove(str(outputfilename)+'_drz_sci.fits') except: print 'its okay, cluster drizzle doesn exist' #Run sextractor on the images tweaksex.tweakreg_sextract(drzList,'catfile') refDate=obsDates[1]# refimage=drzList[1]# drzString= ",".join(drzList) tweaksex.ref_sex( refimage ) tweakreg.TweakReg(drzString, \ residplot='both', \ see2dplot=True, \ threshold=np.float(thresh), \ searchrad=np.float(search_rad), \ fitgeometry='rscale', updatehdr='True', \ refimage=refimage, catfile='catfile', \ xcol=1, ycol=2, fluxcol=3, \ minobj=2, \ refcat='reference.cat', \ refxcol=1, refycol=2, rfluxcol=3) for iDate in uniqueObs: if iDate != obsDates: TweakFLT= np.array(fltList)[ np.array(obsDates) == str(iDate) ] tweakString = ",".join(TweakFLT) print tweakString tweakback.tweakback( str(iDate)+'_drz_sci.fits', \ input=tweakString )
def applyOffset(drzfile, fltfile, hlet=False): """Apply offset to an flt image from an aligned drizzled image""" tweakback.tweakback(drzfile, input=fltfile, origwcs='DRZWCS', verbose=True, force=True) if hlet: headerlet.write_headerlet(fl, d, output=None, sciext='SCI', wcsname='DRZWCS_1', wcskey='PRIMARY', destim=None, sipname=None, npolfile=None, d2imfile=None, author=None, descrip=None, history=None, nmatch=None, catalog=None, attach=True, clobber=False, logging=False)