def fits2fits(infile, outfile, verbose=False, fix_idr=False): """ Returns: error string, or None on success. """ if fix_idr: from astrometry.util.fix_sdss_idr import fix_sdss_idr # Read input file. fitsin = pyfits.open(infile) # Print out info about input file. if verbose: fitsin.info() for i, hdu in enumerate(fitsin): if fix_idr: hdu = fitsin[i] = fix_sdss_idr(hdu) # verify() fails when a keywords contains invalid characters, # so go through the primary header and fix them by converting invalid # characters to '_' hdr = hdu.header logging.info('Header has %i cards' % len(hdr)) # allowed characters (FITS standard section 5.1.2.1) pat = re.compile(r'[^A-Z0-9_\-]') newcards = [] for c in hdr.ascard: k = c.keyword # new keyword: knew = pat.sub('_', k) if k != knew: logging.debug('Replacing illegal keyword %s by %s' % (k, knew)) # it seems pyfits is not clever enough to notice this... if len(knew) > 8: knew = 'HIERARCH ' + knew newcards.append(pyfits.Card(keyword=knew, value=c.value, comment=c.comment)) hdu.header = pyfits.Header(newcards) # Fix input header hdu.verify('fix') # UGH! Work around stupid pyfits handling of scaled data... # (it fails to round-trip scaled data correctly!) bzero = hdr.get('BZERO', None) bscale = hdr.get('BSCALE', None) if (bzero is not None and bscale is not None and (bzero != 0. or bscale != 1.)): logging.debug('Scaling to bzero=%g, bscale=%g' % (bzero, bscale)) hdu.scale('int16', '', bscale, bzero) # Describe output file we're about to write... if verbose: print 'Outputting:' fitsin.info() try: pyfits_writeto(fitsin, outfile, output_verify='warn') except pyfits.VerifyError, ve: return ('Verification of output file failed: your FITS file is probably too broken to automatically fix.' + ' Error message is:' + str(ve))
def removelines(infile, outfile, xcol='X', ycol='Y', ext=1, cut=None, **kwargs): if cut is None: cut = 100 p = pyfits.open(infile) xy = p[ext].data hdr = p[ext].header if xy is None: print 'removelines.py: Input file contains no sources.' pyfits_writeto(p, outfile) return 0 x = xy.field(xcol) y = xy.field(ycol) if len(x) == 0: print 'removelines.py: Your FITS file contains 0 sources (rows)' pyfits_writeto(p, outfile) return 0 ix = hist_remove_lines(x, 1, 0.5, logcut=-cut) iy = hist_remove_lines(y, 1, 0.5, logcut=-cut) I = ix * iy xc = x[I] yc = y[I] print 'removelines.py: Removed %i sources' % (len(x) - len(xc)) p[ext].header.add_history('This xylist was filtered by the "removelines.py" program') p[ext].header.add_history('to remove horizontal and vertical lines of sources') p[ext].header['REMLINEN'] = (len(x) - len(xc), 'Number of sources removed by "removelines.py"') p[ext].data = p[ext].data[I] pyfits_writeto(p, outfile) return 0
def removelines(infile, outfile, xcol='X', ycol='Y', ext=1, cut=None, **kwargs): if cut is None: cut = 100 p = pyfits.open(infile) xy = p[ext].data hdr = p[ext].header if xy is None: print('removelines.py: Input file contains no sources.') pyfits_writeto(p, outfile) return 0 x = xy.field(xcol) y = xy.field(ycol) if len(x) == 0: print('removelines.py: Your FITS file contains 0 sources (rows)') pyfits_writeto(p, outfile) return 0 ix = hist_remove_lines(x, 1, 0.5, logcut=-cut) iy = hist_remove_lines(y, 1, 0.5, logcut=-cut) I = ix * iy xc = x[I] yc = y[I] print('removelines.py: Removed %i sources' % (len(x) - len(xc))) p[ext].header.add_history('This xylist was filtered by the "removelines.py" program') p[ext].header.add_history('to remove horizontal and vertical lines of sources') p[ext].header['REMLINEN'] = (len(x) - len(xc), 'Number of sources removed by "removelines.py"') p[ext].data = p[ext].data[I] pyfits_writeto(p, outfile) return 0
def removelines(infile, outfile, xcol='X', ycol='Y', plots=False, cut=None, **kwargs): p = pyfits.open(infile) xy = p[1].data hdr = p[1].header x = xy.field(xcol) y = xy.field(ycol) NX = max(x) - min(x) NY = max(y) - min(y) nangle = int(ceil(sqrt(NX*NY)/4.)) if cut is None: cut = 20 if plots: clf() plot(x, y, 'r.') I = array([True]*len(x)) for i,angle in enumerate(0.75 + linspace(0, pi/2., nangle, endpoint=False)): cost = cos(angle) sint = sin(angle) xx = x*cost + y*sint yy = x*-sint + y*cost xx -= min(xx) yy -= min(yy) if plots: print clf() subplot(2,2,1) plot(xx, yy, 'r.') subplot(2,2,3) #ix = hist_remove_lines(xx, 0.5, 0.5, 5, plots=plots) ix = hist_remove_lines(xx, 1, 0.5, logcut=-cut) if plots: subplot(2,2,4) #iy = hist_remove_lines(yy, 0.5, 0.5, 5, plots=plots) iy = hist_remove_lines(yy, 1, 0.5, logcut=-cut) I *= ix * iy removed = (ix * iy == False) if plots: if sum(removed): plot([min(x[removed]), max(x[removed])], [min(y[removed]), max(y[removed])], 'k-', alpha=0.5) subplot(2,2,1) plot(xx[removed], yy[removed], 'b-', alpha=0.5) plot(xx[removed], yy[removed], 'b.') savefig('rot-%04i.png' % i) print 'angle', angle, 'removed', (len(x) - sum(ix*iy)) xc = x[I] yc = y[I] print 'removelines.py: Removed %i sources' % (len(x) - len(xc)) if plots: plot(xc, yc, 'o', mec='r', mfc='none') # axes('equal') savefig('after.png') p[1].header.add_history('This xylist was filtered by the "removelines_rotate.py" program') p[1].header.add_history('to remove lines of sources') p[1].header.update('REMLINEN', len(x) - len(xc), 'Number of sources removed by "removelines"') p[1].data = p[1].data[I] pyfits_writeto(p, outfile) return 0
def removelines_general(infile, outfile, nt=180, nr=180, thresh1=2., thresh2=5., plots=False): p = pyfits.open(infile) xy = p[1].data hdr = p[1].header x = xy.field('X').copy() y = xy.field('Y').copy() imshowargs = { 'interpolation':'nearest', 'origin':'lower' } imgw = int(ceil(max(x) - min(x))) imgh = int(ceil(max(y) - min(y))) x -= min(x) y -= min(y) Rmax = sqrt(imgw**2 + imgh**2) Rmin = -Rmax (houghimg, houghnorm, rr, tt, rstep, tstep ) = normalized_hough(x, y, imgw, imgh, Rmin, Rmax, 0, pi, nr, nt) hnorm = houghimg / maximum(houghnorm, 1) if plots: clf() plot(x,y,'r.') savefig('xy.png') clf() imshow(houghimg, **imshowargs) xlabel('Theta') ylabel('Radius') colorbar() savefig('hough.png') clf() imshow(houghnorm, **imshowargs) xlabel('Theta') ylabel('Radius') colorbar() savefig('norm.png') clf() imshow(hnorm, **imshowargs) xlabel('Theta') ylabel('Radius') colorbar() savefig('hnorm.png') I = find(hnorm.ravel() >= thresh1) print '%i peaks are above the coarse threshold' % len(I) bestri = I / nt bestti = I % nt if plots: a=axis() for (ri,ti) in zip(bestri,bestti): plot([ti-2, ti-2, ti+2, ti+2, ti-2], [ri-2, ri+2, ri+2, ri-2, ri-2], 'r-') axis(a) savefig('zooms.png') clf() plot(x,y,'r.') for (ri,ti) in zip(bestri,bestti): (x0,x1,y0,y1) = clip_to_image(rr[ri], tt[ti], imgw, imgh) plot([x0,x1],[y0,y1], 'b-') savefig('xy2.png') # how big a search area around each peak? boxsize = 1 # how much more finely to grid. finer = 3 nr2 = (boxsize * 2)*finer + 1 nt2 = nr2 bestrt = [] keep = array([True] * len(x)) for (ri,ti) in zip(bestri,bestti): (subh, subnorm, subrr, subtt, subrstep, subtstep ) = normalized_hough(x, y, imgw, imgh, rr[max(ri-boxsize, 0)], rr[min(ri+boxsize, nr-1)], tt[max(ti-boxsize, 0)], tt[min(ti+boxsize, nt-1)], nr2, nt2) #print ' median normalization:', median(subnorm) subhnorm = subh / maximum(subnorm,1) I = find((subhnorm).ravel() >= thresh2) for i in I: bestsubri = i / nt2 bestsubti = i % nt2 r = subrr[bestsubri] t = subtt[bestsubti] bestrt.append((r,t)) #print ' (r=%.1f, t=%.1f): factor %.1f above expected' % (r, t*180/pi, subhnorm.ravel()[i]) thisr = x * cos(t) + y * sin(t) keep *= (abs(thisr - r) > subrstep/2.) print 'In finer grid: found %i peaks' % len(bestrt) if plots: clf() subplot(1,1,1) plot(x,y,'r.') for (r,t) in bestrt: (x0,x1,y0,y1) = clip_to_image(r, t, imgw, imgh) plot([x0,x1],[y0,y1],'b-') savefig('xy3.png') clf() plot(x,y,'r.') plot(x[keep == False], y[keep == False], 'b.') savefig('xy4.png') p[1].data = p[1].data[keep] pyfits_writeto(p, outfile) return 0
def removelines_general(infile, outfile, nt=180, nr=180, thresh1=2., thresh2=5., plots=False): p = pyfits.open(infile) xy = p[1].data hdr = p[1].header x = xy.field('X').copy() y = xy.field('Y').copy() imshowargs = {'interpolation': 'nearest', 'origin': 'lower'} imgw = int(ceil(max(x) - min(x))) imgh = int(ceil(max(y) - min(y))) x -= min(x) y -= min(y) Rmax = sqrt(imgw**2 + imgh**2) Rmin = -Rmax (houghimg, houghnorm, rr, tt, rstep, tstep) = normalized_hough(x, y, imgw, imgh, Rmin, Rmax, 0, pi, nr, nt) hnorm = houghimg / maximum(houghnorm, 1) if plots: clf() plot(x, y, 'r.') savefig('xy.png') clf() imshow(houghimg, **imshowargs) xlabel('Theta') ylabel('Radius') colorbar() savefig('hough.png') clf() imshow(houghnorm, **imshowargs) xlabel('Theta') ylabel('Radius') colorbar() savefig('norm.png') clf() imshow(hnorm, **imshowargs) xlabel('Theta') ylabel('Radius') colorbar() savefig('hnorm.png') I = find(hnorm.ravel() >= thresh1) print('%i peaks are above the coarse threshold' % len(I)) bestri = I / nt bestti = I % nt if plots: a = axis() for (ri, ti) in zip(bestri, bestti): plot([ti - 2, ti - 2, ti + 2, ti + 2, ti - 2], [ri - 2, ri + 2, ri + 2, ri - 2, ri - 2], 'r-') axis(a) savefig('zooms.png') clf() plot(x, y, 'r.') for (ri, ti) in zip(bestri, bestti): (x0, x1, y0, y1) = clip_to_image(rr[ri], tt[ti], imgw, imgh) plot([x0, x1], [y0, y1], 'b-') savefig('xy2.png') # how big a search area around each peak? boxsize = 1 # how much more finely to grid. finer = 3 nr2 = (boxsize * 2) * finer + 1 nt2 = nr2 bestrt = [] keep = array([True] * len(x)) for (ri, ti) in zip(bestri, bestti): (subh, subnorm, subrr, subtt, subrstep, subtstep) = normalized_hough( x, y, imgw, imgh, rr[max(ri - boxsize, 0)], rr[min(ri + boxsize, nr - 1)], tt[max(ti - boxsize, 0)], tt[min(ti + boxsize, nt - 1)], nr2, nt2) #print ' median normalization:', median(subnorm) subhnorm = subh / maximum(subnorm, 1) I = find((subhnorm).ravel() >= thresh2) for i in I: bestsubri = i / nt2 bestsubti = i % nt2 r = subrr[bestsubri] t = subtt[bestsubti] bestrt.append((r, t)) #print ' (r=%.1f, t=%.1f): factor %.1f above expected' % (r, t*180/pi, subhnorm.ravel()[i]) thisr = x * cos(t) + y * sin(t) keep *= (abs(thisr - r) > subrstep / 2.) print('In finer grid: found %i peaks' % len(bestrt)) if plots: clf() subplot(1, 1, 1) plot(x, y, 'r.') for (r, t) in bestrt: (x0, x1, y0, y1) = clip_to_image(r, t, imgw, imgh) plot([x0, x1], [y0, y1], 'b-') savefig('xy3.png') clf() plot(x, y, 'r.') plot(x[keep == False], y[keep == False], 'b.') savefig('xy4.png') p[1].data = p[1].data[keep] pyfits_writeto(p, outfile) return 0
def uniformize(infile, outfile, n, xcol='X', ycol='Y', ext=1, **kwargs): p = pyfits.open(infile) xy = p[ext].data if xy is None: print('No sources') pyfits_writeto(p, outfile) return hdr = p[ext].header x = xy.field(xcol) y = xy.field(ycol) if len(x) == 0: print('Empty xylist') pyfits_writeto(p, outfile) return # use IMAGEW,H, or compute bounds? # #$)(*&%^ NaNs in LSST source positions. Seriously, WTF! I = np.logical_and(np.isfinite(x), np.isfinite(y)) if not all(I): print('%i source positions are not finite.' % np.sum(np.logical_not(I))) x = x[I] y = y[I] W = max(x) - min(x) H = max(y) - min(y) if W == 0 or H == 0: print('Area of the rectangle enclosing all image sources: %i x %i' % (W, H)) pyfits_writeto(p, outfile) return NX = int(max(1, np.round(W / np.sqrt(W * H / float(n))))) NY = int(max(1, np.round(n / float(NX)))) print('Uniformizing into %i x %i bins' % (NX, NY)) print('Image bounds: x [%g,%g], y [%g,%g]' % (min(x), max(x), min(y), max(y))) ix = (np.clip(np.floor((x - min(x)) / float(W) * NX), 0, NX - 1)).astype(int) iy = (np.clip(np.floor((y - min(y)) / float(H) * NY), 0, NY - 1)).astype(int) assert (np.all(ix >= 0)) assert (np.all(ix < NX)) assert (np.all(iy >= 0)) assert (np.all(iy < NY)) I = iy * NX + ix assert (np.all(I >= 0)) assert (np.all(I < NX * NY)) bins = [[] for i in range(NX * NY)] for j, i in enumerate(I): bins[int(i)].append(j) maxlen = max([len(b) for b in bins]) J = [] for i in range(maxlen): thisrow = [] for b in bins: if i >= len(b): continue thisrow.append(b[i]) thisrow.sort() J += thisrow J = np.array(J) p[ext].header.add_history( 'This xylist was filtered by the "uniformize.py" program') p[ext].data = p[ext].data[J] pyfits_writeto(p, outfile) return 0