def main(): """Main program. """ import argparse parser = argparse.ArgumentParser( description= "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)." ) parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument( '--byexp', action='store_true', default=False, help='Run one whole exposure per job (not one CCD per job)') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--stage', help='Stage image files to given directory') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--near', action='store_true', help='Quick cut to only CCDs near selected bricks') parser.add_argument('--check-coadd', action='store_true', help='Check which coadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--opt', help='With --command, extra options to add') parser.add_argument('--maxra', type=float, help='Maximum RA to run') parser.add_argument('--minra', type=float, help='Minimum RA to run') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--ignore_cuts', action='store_true', default=False, help='no photometric cuts') parser.add_argument('--save_to_fits', action='store_true', default=False, help='save cut brick,ccd to fits table') parser.add_argument( '--name', action='store', default='dr3', help='save with this suffix, e.g. refers to ccds table') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') parser.add_argument('--nccds', action='store_true', default=False, help='Prints number of CCDs per brick') parser.add_argument('--bands', default='g,r,z', help='Set bands to keep: comma-separated list.') opt = parser.parse_args() want_ccds = (opt.calibs or opt.forced or opt.lsb) want_bricks = not want_ccds survey = LegacySurveyData() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = survey.get_bricks() log('Bricks Dec range:', B.dec.min(), B.dec.max()) if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = survey.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) if opt.ignore_cuts == False: log('Applying CCD cuts...') if 'ccd_cuts' in T.columns(): T.cut(T.ccd_cuts == 0) log(len(T), 'CCDs survive cuts') bands = opt.bands.split(',') log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) log('CCDs Dec range:', T.dec.min(), T.dec.max()) # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo, dhi = -25, 40 rlo, rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict( test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo, dhi = -90, 90 rlo, rhi = 0, 360 elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo, rhi = 240, 245 dlo, dhi = 5, 12 elif opt.region == 'dr8-decam': rlo, rhi = 0, 360 dlo, dhi = -70, 40 log('DR8-DECam region') elif opt.region == 'edrplus': rlo, rhi = 235, 248 dlo, dhi = 5, 15 elif opt.region == 'edr-south': rlo, rhi = 240, 245 dlo, dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo, rhi = 149.75, 150.75 dlo, dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo, rhi = 240, 250 dlo, dhi = 10, 15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo, rhi = 320., 360. dlo, dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo, rhi = 240, 360 dlo, dhi = 20, 40 elif opt.region == 'north': rlo, rhi = 120, 240 dlo, dhi = 20, 40 elif opt.region == 'northeast': rlo, rhi = 0, 120 dlo, dhi = 20, 40 elif opt.region == 'southwest': rlo, rhi = 240, 360 dlo, dhi = -20, 0 elif opt.region == 'south': rlo, rhi = 120, 240 dlo, dhi = -20, 0 elif opt.region == 'southeast': rlo, rhi = 0, 120 dlo, dhi = -20, 0 elif opt.region == 'southsoutheast': rlo, rhi = 0, 120 dlo, dhi = -20, -10 elif opt.region == 'midwest': rlo, rhi = 240, 360 dlo, dhi = 0, 20 elif opt.region == 'middle': rlo, rhi = 120, 240 dlo, dhi = 0, 20 elif opt.region == 'mideast': rlo, rhi = 0, 120 dlo, dhi = 0, 20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo, rhi = 250, 260 dlo, dhi = 30, 35 elif opt.region == 'deep2f2': rlo, rhi = 251.4, 254.4 dlo, dhi = 34.6, 35.3 elif opt.region == 'deep2f3': rlo, rhi = 351.25, 353.75 dlo, dhi = 0, 0.5 elif opt.region == 'deep3': rlo, rhi = 214, 216 dlo, dhi = 52.25, 53.25 elif opt.region == 'virgo': rlo, rhi = 185, 190 dlo, dhi = 10, 15 elif opt.region == 'virgo2': rlo, rhi = 182, 192 dlo, dhi = 8, 18 elif opt.region == 'coma': # van Dokkum et al Coma cluster ultra-diffuse galaxies: 3x3 field centered on Coma cluster rc, dc = 195., 28. dd = 1.5 cosdec = np.cos(np.deg2rad(dc)) rlo, rhi = rc - dd / cosdec, rc + dd / cosdec dlo, dhi = dc - dd, dc + dd elif opt.region == 'lsb': rlo, rhi = 147.2, 147.8 dlo, dhi = -0.4, 0.4 elif opt.region == 'eboss-sgc': # generous boundaries to make sure get all relevant images # RA -45 to +45 # Dec -5 to +7 rlo, rhi = 310., 50. dlo, dhi = -6., 6. elif opt.region == 'eboss-ngc': # generous boundaries to make sure get all relevant images # NGC ELGs # RA 115 to 175 # Dec 15 to 30 # rlo,rhi = 122., 177. # dlo,dhi = 12., 32. rlo, rhi = 126., 168. dlo, dhi = 18., 33. elif opt.region == 'mzls': dlo, dhi = -10., 90. # -10: pull in Stripe 82 data too elif opt.region == 'dr4-bootes': # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched #dlo,dhi = 34., 35. #rlo,rhi = 209.5, 210.5 dlo, dhi = 33., 36. rlo, rhi = 216.5, 219.5 elif opt.region == 'des-sn-x3': #rlo,rhi = 36., 37. #dlo,dhi = -5., -4. rlo, rhi = 36., 36.5 dlo, dhi = -4.5, -4. elif opt.region == 'ngc2632': # open cluster rlo, rhi = 129.0, 131.0 dlo, dhi = 19.0, 20.5 elif opt.region == 'dr8sky': rlo, rhi = 35.0, 37.0 dlo, dhi = -3.0, -1.0 # ADM DR8 test regions, see, e.g.: # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR8#Testregions elif opt.region == 'dr8-test-s82': rlo, rhi = 0, 45 dlo, dhi = -1.25, 1.25 elif opt.region == 'dr8-test-hsc-sgc': rlo, rhi = 30, 40 dlo, dhi = -6.5, -1.25 elif opt.region == 'dr8-test-hsc-ngc': rlo, rhi = 177.5, 182.5 dlo, dhi = -1, 1 elif opt.region == 'dr8-test-edr': rlo, rhi = 240, 245 dlo, dhi = 5, 12 elif opt.region == 'dr8-test-hsc-north': rlo, rhi = 240, 250 dlo, dhi = 42, 45 elif opt.region == 'dr8-test-deep2-egs': rlo, rhi = 213, 216.5 dlo, dhi = 52, 54 elif opt.region == 'dr8-test-overlap': rlo, rhi = 132, 140.5 dlo, dhi = 31.5, 35 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if opt.minra is not None: rlo = opt.minra if opt.maxra is not None: rhi = opt.maxra if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut( np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range; cut Dec range', B.dec.min(), B.dec.max()) #for name in B.get('brickname'): # print(name) #B.writeto('bricks-cut.fits') bricksize = 0.25 # A bit more than 0.25-degree brick radius + Bok image radius ~ 0.57 search_radius = 1.05 * np.sqrt(2.) * (bricksize + (0.455 * 4096 / 3600.)) / 2. log(len(T), 'CCDs') log(len(B), 'Bricks') I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius, nearest=True) B.cut(I) log('Cut to', len(B), 'bricks near CCDs') log('Bricks Dec range:', B.dec.min(), B.dec.max()) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # plt.title('DR3 bricks') # plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1]) # plt.savefig('bricks.png') if opt.touching: I, J, d = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius, nearest=True) # list the ones that will be cut # drop = np.ones(len(T)) # drop[I] = False # for i in np.flatnonzero(drop): # from astrometry.util.starutil_numpy import degrees_between # dists = degrees_between(B.ra, B.dec, T.ra[i], T.dec[i]) # mindist = min(dists) # print('Dropping:', T.ra[i], T.dec[i], 'min dist', mindist, 'search_radius', search_radius) T.cut(I) log('Cut to', len(T), 'CCDs near bricks') # sort by RA increasing B.cut(np.argsort(B.ra)) if opt.save_to_fits: assert (opt.touching) # Write cut tables to file for tab, typ in zip([B, T], ['bricks', 'ccds']): fn = '%s-%s-cut.fits' % (typ, opt.region) if os.path.exists(fn): os.remove(fn) tab.writeto(fn) log('Wrote %s' % fn) # Write text files listing ccd and filename names # nm1,nm2= 'ccds-%s.txt'% opt.region,'filenames-%s.txt' % opt.region # if os.path.exists(nm1): # os.remove(nm1) # if os.path.exists(nm2): # os.remove(nm2) # f1,f2=open(nm1,'w'),open(nm2,'w') # fns= list(set(T.get('image_filename'))) # for fn in fns: # f2.write('%s\n' % fn.strip()) # for ti in T: # f1.write('%s\n' % ti.get('image_filename').strip()) # f1.close() # f2.close() # log('Wrote *-names.txt') if opt.touching: if want_bricks: # Shortcut the list of bricks that are definitely touching CCDs -- # a brick-ccd pair within this radius must be touching. closest_radius = 0.95 * (bricksize + 0.262 * 2048 / 3600.) / 2. J1, nil, nil = match_radec(B.ra, B.dec, T.ra, T.dec, closest_radius, nearest=True) log(len(J1), 'of', len(B), 'bricks definitely touch CCDs') tocheck = np.ones(len(B), bool) tocheck[J1] = False J2 = [] for j in np.flatnonzero(tocheck): b = B[j] wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickname) if len(I) == 0: continue J2.append(j) J = np.hstack((J1, J2)) J = np.sort(J).astype(int) B.cut(J) log('Cut to', len(B), 'bricks touching CCDs') else: J = [] allI = set() for j, b in enumerate(B): wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickname) if len(I) == 0: continue allI.update(I) J.append(j) allI = list(allI) allI.sort() B.cut(np.array(J)) log('Cut to', len(B), 'bricks touching CCDs') elif opt.near: # Find CCDs near bricks allI, nil, nil = match_radec(T.ra, T.dec, B.ra, B.dec, search_radius, nearest=True) # Find bricks near CCDs J, nil, nil = match_radec(B.ra, B.dec, T.ra, T.dec, search_radius, nearest=True) B.cut(J) log('Cut to', len(B), 'bricks near CCDs') else: allI = np.arange(len(T)) if opt.byexp: nil, eI = np.unique(T.expnum[allI], return_index=True) allI = allI[eI] print('Cut to', len(allI), 'expnums') if opt.nccds: from queue import Queue from threading import Thread log('Checking number of CCDs per brick') def worker(): while True: i = q.get() if i is None: break b = B[i] wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(b.brickname, len(I)) q.task_done() q = Queue() num_threads = 24 threads = [] for i in range(num_threads): t = Thread(target=worker) t.start() threads.append(t) for i in range(len(B)): q.put(i) q.join() for i in range(num_threads): q.put(None) for t in threads: t.join() if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) if want_bricks: # Print the list of bricks and exit. for b in B: print(b.brickname) if opt.save_to_fits: B.writeto('bricks-%s-touching.fits' % opt.region) if not want_ccds: sys.exit(0) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.stage is not None: cmd_pat = 'rsync -LRarv %s %s' fns = set() for iccd in allI: im = survey.get_image_object(T[iccd]) fns.update([ im.imgfn, im.wtfn, im.dqfn, im.psffn, im.merged_psffn, im.merged_splineskyfn, im.splineskyfn ]) for i, fn in enumerate(fns): print('File', i + 1, 'of', len(fns), ':', fn) if not os.path.exists(fn): print('No such file:', fn) continue base = survey.get_survey_dir() if base.endswith('/'): base = base[:-1] rel = os.path.relpath(fn, base) dest = os.path.join(opt.stage, rel) print('Dest:', dest) if os.path.exists(dest): print('Exists:', dest) continue cmd = cmd_pat % ('%s/./%s' % (base, rel), opt.stage) print(cmd) rtn = os.system(cmd) assert (rtn == 0) sys.exit(0) if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): expstr = '%08i' % T.expnum[i] imgfn = os.path.join(survey.survey_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] outfn = os.path.join( expstr[:5], expstr, 'forced-%s-%s-%s.fits' % (T.camera[i].strip(), expstr, T.ccdname[i])) f.write( 'python legacypipe/forced_photom.py --apphot --derivs --catalog-dir /project/projectdirs/cosmo/data/legacysurvey/dr7/ %i %s forced/%s\n' % (T.expnum[i], T.ccdname[i], outfn)) f.close() log('Wrote', opt.out) fn = 'forced-ccds.fits' T[allI].writeto(fn) print('Wrote', fn) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write( 'python legacyanalysis/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if cmd is None: cmd = '' f.write(cmd + ' '.join(batch) + '\n') cmd = None if opt.command: cmd = 'python legacypipe/run-calib.py ' if opt.opt is not None: cmd += opt.opt + ' ' for j, i in enumerate(allI): if opt.delete_sky: log(j + 1, 'of', len(allI)) im = survey.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.command: if opt.byexp: s = '--expnum %i' % (T.expnum[i]) else: s = '%i-%s' % (T.expnum[i], T.ccdname[i]) prefix = 'python legacypipe/run-calib.py ' if opt.opt is not None: prefix = prefix + opt.opt #('python legacypipe/run-calib.py --expnum %i --ccdname %s' % # (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] prefix = '' if j < 10: print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i], 'filename', T.image_filename[i]) if not opt.nper: f.write(prefix + s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, cmd) batch = [] if len(batch): write_batch(f, batch, cmd) f.close() log('Wrote', opt.out) return 0
def main(): import argparse parser = argparse.ArgumentParser( description='This script creates small self-contained data sets that ' 'are useful for test cases of the pipeline codes.') parser.add_argument('ccds', help='CCDs table describing region to grab') parser.add_argument('outdir', help='Output directory name') parser.add_argument('brick', help='Brick containing these images') parser.add_argument('--wise', help='For WISE outputs, give the path to a WCS file describing the sub-brick region of interest, eg, a coadd image') parser.add_argument('--fpack', action='store_true', default=False) parser.add_argument('--pad', action='store_true', default=False, help='Keep original image size, but zero out pixels outside ROI') args = parser.parse_args() C = fits_table(args.ccds) print(len(C), 'CCDs in', args.ccds) C.camera = np.array([c.strip() for c in C.camera]) survey = LegacySurveyData() bricks = survey.get_bricks_readonly() outbricks = bricks[np.array([n == args.brick for n in bricks.brickname])] assert(len(outbricks) == 1) outsurvey = LegacySurveyData(survey_dir = args.outdir) trymakedirs(args.outdir) outbricks.writeto(os.path.join(args.outdir, 'survey-bricks.fits.gz')) targetwcs = wcs_for_brick(outbricks[0]) H,W = targetwcs.shape tycho = fits_table(os.path.join(survey.get_survey_dir(), 'tycho2.fits.gz')) print('Read', len(tycho), 'Tycho-2 stars') ok,tx,ty = targetwcs.radec2pixelxy(tycho.ra, tycho.dec) margin = 100 tycho.cut(ok * (tx > -margin) * (tx < W+margin) * (ty > -margin) * (ty < H+margin)) print('Cut to', len(tycho), 'Tycho-2 stars within brick') del ok,tx,ty tycho.writeto(os.path.join(args.outdir, 'tycho2.fits.gz')) outccds = C.copy() for c in ['ccd_x0', 'ccd_x1', 'ccd_y0', 'ccd_y1', 'brick_x0', 'brick_x1', 'brick_y0', 'brick_y1', 'plver', 'skyver', 'wcsver', 'psfver', 'skyplver', 'wcsplver', 'psfplver' ]: outccds.delete_column(c) outccds.image_hdu[:] = 1 # Convert to list to avoid truncating filenames outccds.image_filename = [fn for fn in outccds.image_filename] for iccd,ccd in enumerate(C): decam = (ccd.camera.strip() == 'decam') bok = (ccd.camera.strip() == '90prime') im = survey.get_image_object(ccd) print('Got', im) slc = (slice(ccd.ccd_y0, ccd.ccd_y1), slice(ccd.ccd_x0, ccd.ccd_x1)) tim = im.get_tractor_image(slc, pixPsf=True, splinesky=True, subsky=False, nanomaggies=False) print('Tim:', tim.shape) psf = tim.getPsf() print('PSF:', psf) psfex = psf.psfex print('PsfEx:', psfex) outim = outsurvey.get_image_object(ccd) print('Output image:', outim) print('Image filename:', outim.imgfn) trymakedirs(outim.imgfn, dir=True) imgdata = tim.getImage() dqdata = tim.dq if decam: # DECam-specific code remaps the DQ codes... re-read raw print('Reading data quality from', im.dqfn, 'hdu', im.hdu) dqdata = im._read_fits(im.dqfn, im.hdu, slice=tim.slice) ivdata = tim.getInvvar() if args.pad: # Create zero image of full size, copy in data. fullsize = np.zeros((ccd.height, ccd.width), imgdata.dtype) fullsize[slc] = imgdata imgdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), dqdata.dtype) fullsize[slc] = dqdata dqdata = fullsize fullsize = np.zeros((ccd.height, ccd.width), ivdata.dtype) fullsize[slc] = ivdata ivdata = fullsize else: # Adjust the header WCS by x0,y0 crpix1 = tim.hdr['CRPIX1'] crpix2 = tim.hdr['CRPIX2'] tim.hdr['CRPIX1'] = crpix1 - ccd.ccd_x0 tim.hdr['CRPIX2'] = crpix2 - ccd.ccd_y0 # Add image extension to filename # fitsio doesn't compress .fz by default, so drop .fz suffix outim.imgfn = outim.imgfn.replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.imgfn = outim.imgfn.replace('.fits.fz', '.fits') # if bok: # outim.whtfn = outim.whtfn .replace('.wht.fits', '-%s.wht.fits' % im.ccdname) # if not args.fpack: # outim.whtfn = outim.whtfn .replace('.fits.fz', '.fits') # else: if True: outim.wtfn = outim.wtfn .replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.wtfn = outim.wtfn .replace('.fits.fz', '.fits') if outim.dqfn is not None: outim.dqfn = outim.dqfn .replace('.fits', '-%s.fits' % im.ccdname) if not args.fpack: outim.dqfn = outim.dqfn .replace('.fits.fz', '.fits') if bok: outim.psffn = outim.psffn.replace('.psf', '-%s.psf' % im.ccdname) ccdfn = outim.imgfn ccdfn = ccdfn.replace(outsurvey.get_image_dir(),'') if ccdfn.startswith('/'): ccdfn = ccdfn[1:] outccds.image_filename[iccd] = ccdfn print('Changed output filenames to:') print(outim.imgfn) print(outim.dqfn) ofn = outim.imgfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fitsio.write(ofn, None, header=tim.primhdr, clobber=True) fitsio.write(ofn, imgdata, header=tim.hdr, extname=ccd.ccdname) if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, outim.imgfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) h,w = tim.shape if not args.pad: outccds.width[iccd] = w outccds.height[iccd] = h outccds.crpix1[iccd] = crpix1 - ccd.ccd_x0 outccds.crpix2[iccd] = crpix2 - ccd.ccd_y0 wcs = Tan(*[float(x) for x in [ccd.crval1, ccd.crval2, ccd.crpix1, ccd.crpix2, ccd.cd1_1, ccd.cd1_2, ccd.cd2_1, ccd.cd2_2, ccd.width, ccd.height]]) if args.pad: subwcs = wcs else: subwcs = wcs.get_subimage(ccd.ccd_x0, ccd.ccd_y0, w, h) outccds.ra[iccd],outccds.dec[iccd] = subwcs.radec_center() #if not bok: if True: print('Weight filename:', outim.wtfn) wfn = outim.wtfn # else: # print('Weight filename:', outim.whtfn) # wfn = outim.whtfn trymakedirs(wfn, dir=True) ofn = wfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fitsio.write(ofn, None, header=tim.primhdr, clobber=True) fitsio.write(ofn, ivdata, header=tim.hdr, extname=ccd.ccdname) if args.fpack: cmd = 'fpack -qz 8 -S %s > %s && rm %s' % (ofn, wfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) if outim.dqfn is not None: print('DQ filename', outim.dqfn) trymakedirs(outim.dqfn, dir=True) ofn = outim.dqfn if args.fpack: f,ofn = tempfile.mkstemp(suffix='.fits') os.close(f) fitsio.write(ofn, None, header=tim.primhdr, clobber=True) fitsio.write(ofn, dqdata, header=tim.hdr, extname=ccd.ccdname) if args.fpack: cmd = 'fpack -g -q 0 -S %s > %s && rm %s' % (ofn, outim.dqfn, ofn) print('Running:', cmd) rtn = os.system(cmd) assert(rtn == 0) print('PSF filename:', outim.psffn) trymakedirs(outim.psffn, dir=True) psfex.writeto(outim.psffn) if not bok: print('Sky filename:', outim.splineskyfn) sky = tim.getSky() print('Sky:', sky) trymakedirs(outim.splineskyfn, dir=True) sky.write_fits(outim.splineskyfn) outccds.writeto(os.path.join(args.outdir, 'survey-ccds-1.fits.gz')) # WISE if args.wise is not None: from wise.forcedphot import unwise_tiles_touching_wcs from wise.unwise import (unwise_tile_wcs, unwise_tiles_touching_wcs, get_unwise_tractor_image, get_unwise_tile_dir) # Read WCS... print('Reading TAN wcs header from', args.wise) targetwcs = Tan(args.wise) tiles = unwise_tiles_touching_wcs(targetwcs) print('Cut to', len(tiles), 'unWISE tiles') H,W = targetwcs.shape r,d = targetwcs.pixelxy2radec(np.array([1, W, W/2, W/2]), np.array([H/2, H/2, 1, H ])) roiradec = [r[0], r[1], d[2], d[3]] unwise_dir = os.environ['UNWISE_COADDS_DIR'] wise_out = os.path.join(args.outdir, 'images', 'unwise') print('Will write WISE outputs to', wise_out) unwise_tr_dir = os.environ['UNWISE_COADDS_TIMERESOLVED_DIR'] wise_tr_out = os.path.join(args.outdir, 'images', 'unwise-tr') print('Will write WISE time-resolved outputs to', wise_tr_out) W = fits_table(os.path.join(unwise_tr_dir, 'time_resolved_neo1-atlas.fits')) print('Read', len(W), 'time-resolved WISE coadd tiles') W.cut(np.array([t in tiles.coadd_id for t in W.coadd_id])) print('Cut to', len(W), 'time-resolved vs', len(tiles), 'full-depth') # Write the time-resolved index subset. W.writeto(os.path.join(wise_tr_out, 'time_resolved_neo1-atlas.fits')) # this ought to be enough for anyone =) Nepochs = 5 wisedata = [] # full depth for band in [1,2,3,4]: wisedata.append((unwise_dir, wise_out, tiles.coadd_id, band)) # time-resolved for band in [1,2]: # W1 is bit 0 (value 0x1), W2 is bit 1 (value 0x2) bitmask = (1 << (band-1)) for e in range(Nepochs): # Which tiles have images for this epoch? I = np.flatnonzero(W.epoch_bitmask[:,e] & bitmask) if len(I) == 0: continue print('Epoch %i: %i tiles:' % (e, len(I)), W.coadd_id[I]) edir = os.path.join(unwise_tr_dir, 'e%03i' % e) eoutdir = os.path.join(wise_tr_out, 'e%03i' % e) wisedata.append((edir, eoutdir, tiles.coadd_id[I], band)) wrote_masks = set() for indir, outdir, tiles, band in wisedata: for tile in tiles: wanyband = 'w' tim = get_unwise_tractor_image(indir, tile, band, bandname=wanyband, roiradecbox=roiradec) print('Got unWISE tim', tim) print(tim.shape) thisdir = get_unwise_tile_dir(outdir, tile) print('Directory for this WISE tile:', thisdir) base = os.path.join(thisdir, 'unwise-%s-w%i-' % (tile, band)) print('Base filename:', base) masked = True mu = 'm' if masked else 'u' imfn = base + 'img-%s.fits' % mu ivfn = base + 'invvar-%s.fits.gz' % mu nifn = base + 'n-%s.fits.gz' % mu nufn = base + 'n-u.fits.gz' #print('WISE image header:', tim.hdr) # Adjust the header WCS by x0,y0 wcs = tim.wcs.wcs tim.hdr['CRPIX1'] = wcs.crpix[0] tim.hdr['CRPIX2'] = wcs.crpix[1] H,W = tim.shape tim.hdr['IMAGEW'] = W tim.hdr['IMAGEH'] = H print('WCS:', wcs) print('Header CRPIX', tim.hdr['CRPIX1'], tim.hdr['CRPIX2']) trymakedirs(imfn, dir=True) fitsio.write(imfn, tim.getImage(), header=tim.hdr, clobber=True) print('Wrote', imfn) fitsio.write(ivfn, tim.getInvvar(), header=tim.hdr, clobber=True) print('Wrote', ivfn) fitsio.write(nifn, tim.nims, header=tim.hdr, clobber=True) print('Wrote', nifn) fitsio.write(nufn, tim.nuims, header=tim.hdr, clobber=True) print('Wrote', nufn) if not (indir,tile) in wrote_masks: print('Looking for mask file for', indir, tile) # record that we tried this dir/tile combo wrote_masks.add((indir,tile)) for idir in indir.split(':'): tdir = get_unwise_tile_dir(idir, tile) maskfn = 'unwise-%s-msk.fits.gz' % tile fn = os.path.join(tdir, maskfn) print('Mask file:', fn) if os.path.exists(fn): print('Reading', fn) (x0,x1,y0,y1) = tim.roi roislice = (slice(y0,y1), slice(x0,x1)) F = fitsio.FITS(fn)[0] hdr = F.read_header() M = F[roislice] outfn = os.path.join(thisdir, maskfn) fitsio.write(outfn, M, header=tim.hdr, clobber=True) print('Wrote', outfn) break outC = outsurvey.get_ccds_readonly() for iccd,ccd in enumerate(outC): outim = outsurvey.get_image_object(ccd) print('Got output image:', outim) otim = outim.get_tractor_image(pixPsf=True, splinesky=True) print('Got output tim:', otim)
def main(survey=None, opt=None): '''Driver function for forced photometry of individual DECam images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0,x1,y0,y1) = opt.zoom zoomslice = (slice(y0,y1), slice(x0,x1)) ps = None if opt.plots is not None: from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.filename) opt.filename = None except: # make this 'None' for survey.find_ccds() expnum = None # Try parsing HDU number try: opt.hdu = int(opt.hdu) ccdname = None except: ccdname = opt.hdu opt.hdu = -1 if survey is None: survey = LegacySurveyData() if opt.filename is not None and opt.hdu >= 0: # Read metadata from file T = exposure_metadata([opt.filename], hdus=[opt.hdu]) print('Metadata:') T.about() else: # Read metadata from survey-ccds.fits table T = survey.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if opt.hdu >= 0: T.cut(T.image_hdu == opt.hdu) print(len(T), 'with HDU', opt.hdu) if opt.filename is not None: T.cut(np.array([f.strip() == opt.filename for f in T.image_filename])) print(len(T), 'with filename', opt.filename) assert(len(T) == 1) ccd = T[0] im = survey.get_image_object(ccd) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True, constant_invvar=opt.constant_invvar) print('Got tim:', tim) print('Read image:', Time()-t0) if opt.catfn in ['DR1', 'DR2', 'DR3']: margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, survey=survey) for b in bricks: # there is some overlap with this brick... read the catalog. fn = survey.find_file('tractor', brick=b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok,xx,yy = chipwcs.radec2pixelxy(T.ra, T.dec) W,H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W+margin)) * (yy >= -margin) * (yy <= (H+margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') T.cut((T.out_of_bounds == False) * (T.left_blob == False)) print('Cut to', len(T), 'on out_of_bounds and left_blob') if len(T): TT.append(T) if len(TT) == 0: print('No sources to photometer.') return 0 T = merge_tables(TT, columns='fillzero') T._header = TT[0]._header del TT # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero(np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) surveydir = survey.get_survey_dir() del survey cat = read_fits_catalog(T) # print('Got cat:', cat) print('Read catalog:', Time()-t0) print('Forced photom...') opti = None forced_kwargs = {} if opt.ceres: from tractor.ceres_optimizer import CeresOptimizer B = 8 opti = CeresOptimizer(BW=B, BH=B) #forced_kwargs.update(verbose=True) for src in cat: # Limit sizes of huge models from tractor.galaxy import ProfileGalaxy if isinstance(src, ProfileGalaxy): px,py = tim.wcs.positionToPixel(src.getPosition()) h = src._getUnitFluxPatchSize(tim, px, py, tim.modelMinval) MAXHALF = 128 if h > MAXHALF: print('halfsize', h,'for',src,'-> setting to',MAXHALF) src.halfsize = MAXHALF tr = Tractor([tim], cat, optimizer=opti) tr.freezeParam('images') for src in cat: src.freezeAllBut('brightness') src.getBrightness().freezeAllBut(tim.band) disable_galaxy_cache() F = fits_table() F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32) ok,x,y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x-1).astype(np.float32) F.y = (y-1).astype(np.float32) if opt.forced: if opt.plots is None: forced_kwargs.update(wantims=False) R = tr.optimize_forced_photometry(variance=True, fitstats=True, shared_params=False, priors=False, **forced_kwargs) if opt.plots: (data,mod,ie,chi,roi) = R.ims1[0] ima = tim.ima imchi = dict(interpolation='nearest', origin='lower', vmin=-5, vmax=5) plt.clf() plt.imshow(data, **ima) plt.title('Data: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(mod, **ima) plt.title('Model: %s' % tim.name) ps.savefig() plt.clf() plt.imshow(chi, **imchi) plt.title('Chi: %s' % tim.name) ps.savefig() F.flux = np.array([src.getBrightness().getFlux(tim.band) for src in cat]).astype(np.float32) F.flux_ivar = R.IV.astype(np.float32) F.fracflux = R.fitstats.profracflux.astype(np.float32) F.rchi2 = R.fitstats.prochi2 .astype(np.float32) print('Forced photom:', Time()-t0) if opt.apphot: import photutils img = tim.getImage() ie = tim.getInvError() with np.errstate(divide='ignore'): imsigma = 1. / ie imsigma[ie == 0] = 0. apimg = [] apimgerr = [] # Aperture photometry locations xxyy = np.vstack([tim.wcs.positionToPixel(src.getPosition()) for src in cat]).T apxy = xxyy - 1. apertures = apertures_arcsec / tim.wcs.pixel_scale() print('Apertures:', apertures, 'pixels') for rad in apertures: aper = photutils.CircularAperture(apxy, rad) p = photutils.aperture_photometry(img, aper, error=imsigma) apimg.append(p.field('aperture_sum')) apimgerr.append(p.field('aperture_sum_err')) ap = np.vstack(apimg).T ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux = ap.astype(np.float32) ap = 1./(np.vstack(apimgerr).T)**2 ap[np.logical_not(np.isfinite(ap))] = 0. F.apflux_ivar = ap.astype(np.float32) print('Aperture photom:', Time()-t0) program_name = sys.argv[0] version_hdr = get_version_header(program_name, surveydir) filename = getattr(ccd, 'image_filename') if filename is None: # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) filename = os.path.join(d2, d1, fname) print('Trimmed filename to', filename) version_hdr.add_record(dict(name='CPFILE', value=filename, comment='CP file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext')) version_hdr.add_record(dict(name='CAMERA', value=ccd.camera, comment='Camera')) version_hdr.add_record(dict(name='EXPNUM', value=im.expnum, comment='Exposure num')) version_hdr.add_record(dict(name='CCDNAME', value=im.ccdname, comment='CCD name')) version_hdr.add_record(dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record(dict(name='EXPOSURE', value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname), comment='Name of this image')) keys = ['TELESCOP','OBSERVAT','OBS-LAT','OBS-LONG','OBS-ELEV', 'INSTRUME'] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = {'exptime':'sec', 'flux':'nanomaggy', 'flux_ivar':'1/nanomaggy^2'} columns = F.get_columns() for i,col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i+1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) if opt.save_model or opt.save_data: hdr = fitsio.FITSHDR() tim.getWcs().wcs.add_to_header(hdr) if opt.save_model: print('Getting model image...') mod = tr.getModelImage(tim) fitsio.write(opt.save_model, mod, header=hdr, clobber=True) print('Wrote', opt.save_model) if opt.save_data: fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True) print('Wrote', opt.save_data) print('Finished forced phot:', Time()-t0) return 0
def main(survey=None, opt=None): '''Driver function for forced photometry of individual Legacy Survey images. ''' if opt is None: parser = get_parser() opt = parser.parse_args() Time.add_measurement(MemMeas) t0 = Time() if os.path.exists(opt.outfn): print('Ouput file exists:', opt.outfn) sys.exit(0) if opt.derivs and opt.agn: print('Sorry, can\'t do --derivs AND --agn') sys.exit(0) if not opt.forced: opt.apphot = True zoomslice = None if opt.zoom is not None: (x0, x1, y0, y1) = opt.zoom zoomslice = (slice(y0, y1), slice(x0, x1)) ps = None if opt.plots is not None: import pylab as plt from astrometry.util.plotutils import PlotSequence ps = PlotSequence(opt.plots) # Try parsing filename as exposure number. try: expnum = int(opt.expnum) filename = None except: # make this 'None' for survey.find_ccds() expnum = None filename = opt.expnum # Try parsing HDU number try: hdu = int(opt.ccdname) ccdname = None except: hdu = -1 ccdname = opt.ccdname if survey is None: survey = LegacySurveyData() catsurvey = survey if opt.catalog_dir is not None: catsurvey = LegacySurveyData(survey_dir=opt.catalog_dir) if filename is not None and hdu >= 0: # FIXME -- try looking up in CCDs file? # Read metadata from file print('Warning: faking metadata from file contents') T = exposure_metadata([filename], hdus=[hdu]) print('Metadata:') T.about() if not 'ccdzpt' in T.columns(): phdr = fitsio.read_header(filename) T.ccdzpt = np.array([phdr['MAGZERO']]) print('WARNING: using header MAGZERO') T.ccdraoff = np.array([0.]) T.ccddecoff = np.array([0.]) print('WARNING: setting CCDRAOFF, CCDDECOFF to zero.') else: # Read metadata from survey-ccds.fits table T = survey.find_ccds(expnum=expnum, ccdname=ccdname) print(len(T), 'with expnum', expnum, 'and CCDname', ccdname) if hdu >= 0: T.cut(T.image_hdu == hdu) print(len(T), 'with HDU', hdu) if filename is not None: T.cut(np.array([f.strip() == filename for f in T.image_filename])) print(len(T), 'with filename', filename) if opt.camera is not None: T.cut(T.camera == opt.camera) print(len(T), 'with camera', opt.camera) assert (len(T) == 1) ccd = T[0] im = survey.get_image_object(ccd) if opt.do_calib: #from legacypipe.survey import run_calibs #kwa = dict(splinesky=True) #run_calibs((im, kwa)) im.run_calibs(splinesky=True) tim = im.get_tractor_image(slc=zoomslice, pixPsf=True, splinesky=True, constant_invvar=opt.constant_invvar, hybridPsf=opt.hybrid_psf, normalizePsf=opt.normalize_psf) print('Got tim:', tim) print('Read image:', Time() - t0) if opt.catfn in ['DR1', 'DR2', 'DR3', 'DR5', 'DR']: margin = 20 TT = [] chipwcs = tim.subwcs bricks = bricks_touching_wcs(chipwcs, survey=catsurvey) for b in bricks: # there is some overlap with this brick... read the catalog. fn = catsurvey.find_file('tractor', brick=b.brickname) if not os.path.exists(fn): print('WARNING: catalog', fn, 'does not exist. Skipping!') continue print('Reading', fn) T = fits_table(fn) ok, xx, yy = chipwcs.radec2pixelxy(T.ra, T.dec) W, H = chipwcs.get_width(), chipwcs.get_height() I = np.flatnonzero((xx >= -margin) * (xx <= (W + margin)) * (yy >= -margin) * (yy <= (H + margin))) T.cut(I) print('Cut to', len(T), 'sources within image + margin') # print('Brick_primary:', np.unique(T.brick_primary)) T.cut(T.brick_primary) print('Cut to', len(T), 'on brick_primary') for col in ['out_of_bounds', 'left_blob']: if col in T.get_columns(): T.cut(T.get(col) == False) print('Cut to', len(T), 'on', col) if len(T): TT.append(T) if len(TT) == 0: print('No sources to photometer.') return 0 T = merge_tables(TT, columns='fillzero') T._header = TT[0]._header del TT print('Total of', len(T), 'catalog sources') # Fix up various failure modes: # FixedCompositeGalaxy(pos=RaDecPos[240.51147402832561, 10.385488075518923], brightness=NanoMaggies: g=(flux -2.87), r=(flux -5.26), z=(flux -7.65), fracDev=FracDev(0.60177207), shapeExp=re=3.78351e-44, e1=9.30367e-13, e2=1.24392e-16, shapeDev=re=inf, e1=-0, e2=-0) # -> convert to EXP I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapedev_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to EXP') for i in I: T.type[i] = 'EXP' # Same thing with the exp component. # -> convert to DEV I = np.flatnonzero( np.array([((t.type == 'COMP') and (not np.isfinite(t.shapeexp_r))) for t in T])) if len(I): print('Converting', len(I), 'bogus COMP galaxies to DEV') for i in I: T.type[i] = 'DEV' if opt.write_cat: T.writeto(opt.write_cat) print('Wrote catalog to', opt.write_cat) else: T = fits_table(opt.catfn) surveydir = survey.get_survey_dir() del survey kwargs = {} cols = T.get_columns() if 'flux_r' in cols and not 'decam_flux_r' in cols: kwargs.update(fluxPrefix='') cat = read_fits_catalog(T, **kwargs) # Replace the brightness (which will be a NanoMaggies with g,r,z) # with a NanoMaggies with this image's band only. for src in cat: src.brightness = NanoMaggies(**{tim.band: 1.}) print('Read catalog:', Time() - t0) print('Forced photom...') F = run_forced_phot(cat, tim, ceres=opt.ceres, derivs=opt.derivs, fixed_also=True, agn=opt.agn, do_forced=opt.forced, do_apphot=opt.apphot, ps=ps) t0 = Time() F.release = T.release F.brickid = T.brickid F.brickname = T.brickname F.objid = T.objid F.camera = np.array([ccd.camera] * len(F)) F.expnum = np.array([im.expnum] * len(F)).astype(np.int32) F.ccdname = np.array([im.ccdname] * len(F)) # "Denormalizing" F.filter = np.array([tim.band] * len(T)) F.mjd = np.array([tim.primhdr['MJD-OBS']] * len(T)) F.exptime = np.array([tim.primhdr['EXPTIME']] * len(T)).astype(np.float32) F.ra = T.ra F.dec = T.dec ok, x, y = tim.sip_wcs.radec2pixelxy(T.ra, T.dec) F.x = (x - 1).astype(np.float32) F.y = (y - 1).astype(np.float32) h, w = tim.shape F.mask = tim.dq[np.clip(np.round(F.y).astype(int), 0, h - 1), np.clip(np.round(F.x).astype(int), 0, w - 1)] program_name = sys.argv[0] version_hdr = get_version_header(program_name, surveydir) filename = getattr(ccd, 'image_filename') if filename is None: # HACK -- print only two directory names + filename of CPFILE. fname = os.path.basename(im.imgfn) d = os.path.dirname(im.imgfn) d1 = os.path.basename(d) d = os.path.dirname(d) d2 = os.path.basename(d) filename = os.path.join(d2, d1, fname) print('Trimmed filename to', filename) version_hdr.add_record( dict(name='CPFILE', value=filename, comment='CP file')) version_hdr.add_record(dict(name='CPHDU', value=im.hdu, comment='CP ext')) version_hdr.add_record( dict(name='CAMERA', value=ccd.camera, comment='Camera')) version_hdr.add_record( dict(name='EXPNUM', value=im.expnum, comment='Exposure num')) version_hdr.add_record( dict(name='CCDNAME', value=im.ccdname, comment='CCD name')) version_hdr.add_record( dict(name='FILTER', value=tim.band, comment='Bandpass of this image')) version_hdr.add_record( dict(name='EXPOSURE', value='%s-%s-%s' % (ccd.camera, im.expnum, im.ccdname), comment='Name of this image')) keys = [ 'TELESCOP', 'OBSERVAT', 'OBS-LAT', 'OBS-LONG', 'OBS-ELEV', 'INSTRUME' ] for key in keys: if key in tim.primhdr: version_hdr.add_record(dict(name=key, value=tim.primhdr[key])) hdr = fitsio.FITSHDR() units = { 'exptime': 'sec', 'flux': 'nanomaggy', 'flux_ivar': '1/nanomaggy^2' } columns = F.get_columns() for i, col in enumerate(columns): if col in units: hdr.add_record(dict(name='TUNIT%i' % (i + 1), value=units[col])) outdir = os.path.dirname(opt.outfn) if len(outdir): trymakedirs(outdir) fitsio.write(opt.outfn, None, header=version_hdr, clobber=True) F.writeto(opt.outfn, header=hdr, append=True) print('Wrote', opt.outfn) if opt.save_model or opt.save_data: hdr = fitsio.FITSHDR() tim.getWcs().wcs.add_to_header(hdr) if opt.save_model: print('Getting model image...') tr = Tractor([tim], cat) mod = tr.getModelImage(tim) fitsio.write(opt.save_model, mod, header=hdr, clobber=True) print('Wrote', opt.save_model) if opt.save_data: fitsio.write(opt.save_data, tim.getImage(), header=hdr, clobber=True) print('Wrote', opt.save_data) print('Finished forced phot:', Time() - t0) return 0