Example #1
0
def distmatch(sexlist,
              catlist,
              maxrad=180,
              minrad=10,
              reqmatch=3,
              patolerance=1.2,
              uncpa=-1,
              showmatches=0,
              fastmatch=1):

    if reqmatch < 2:
        print 'Warning: reqmatch >=3 suggested'
    if patolerance <= 0:
        print 'PA tolerance cannot be negative!!!'
        patolerance = abs(patolerance)
    if uncpa < 0: uncpa = 720

    declist = []
    for s in sexlist:
        declist.append(s.dec_rad)
    avdec_rad = astrometrystats.median(declist)  # faster distance computation
    rascale = numpy.cos(avdec_rad)  # will mess up meridian crossings, however

    #Calculates distances between objects in same list for both image catalog and reference catalog
    (sexdists, sexmatchids) = calcdist(sexlist, maxrad, minrad, rascale)
    (catdists, catmatchids) = calcdist(catlist, maxrad, minrad, rascale)

    # Now look for matches in the reference catalog to distances in the image catalog.
    countgreatmatches = 0

    smatch = []
    cmatch = []
    mpa = []
    offset = []
    offpa = []
    nmatch = []
    primarymatchs = []
    primarymatchc = []

    #For each sextractor source A
    for si in range(len(sexdists)):
        sexdistarr = sexdists[si]
        sexidarr = sexmatchids[si]
        if len(sexdistarr) < 2: continue

        #For each catalog source B
        for ci in range(len(catdists)):
            catdistarr = catdists[ci]
            catidarr = catmatchids[ci]
            if len(catdistarr) < 2: continue
            match = 0
            smatchin = []
            cmatchin = []

            #For each sextractor source A, use the distances to other sextractor sources
            #that are within the radius range
            for sj in range(len(sexdistarr)):
                sexdist = sexdistarr[sj]
                newmatch = 1

                #For each catalog source B, use the distances to other catalog sources
                #that are within the radius range and compare the ratio to
                #(sextractor source A - sextractor sources) to (catalog source B - catalog sources)
                #If within 3% then save the match (only count 1 match per catalog row)
                for cj in range(len(catdistarr)):
                    catdist = catdistarr[cj]
                    if abs((sexdist / catdist) - 1.0) < 0.03:

                        match += newmatch
                        newmatch = 0  #further matches before the next sj loop indicate degeneracies
                        smatchin.append(sexmatchids[si][sj])
                        cmatchin.append(catmatchids[ci][cj])

#If number of matches is above or equal to required number of matches for sextractor source A
#(i.e. sextractor source A's distance to other sources makes it a likely candidate to be a catalog source)
#Find difference in position angle between matched angle between sextractor source A and matched source with
#catalog source B and matched catalog source.
#Remove values that are larger than user specified value or if above position angle tolerance.
            if match >= reqmatch:

                dpa = []

                # Here, dpa[n] is the mean rotation of the PA from the primary star of this match
                #  to the stars in its match RELATIVE TO those same angles for those same stars
                #  in the catalog.  Therefore it is a robust measurement of the rotation.
                for i in range(len(smatchin)):
                    ddpa = posangle(sexlist[si],
                                    sexlist[smatchin[i]]) - posangle(
                                        catlist[ci], catlist[cmatchin[i]])
                    while ddpa > 200:
                        ddpa -= 360.
                    while ddpa < -160:
                        ddpa += 360.
                    dpa.append(ddpa)

                #If user was confident the initial PA was right, remove bad PA's right away
                for i in range(len(smatchin) - 1, -1, -1):
                    if abs(dpa[i]) > uncpa:
                        del smatchin[i]
                        del cmatchin[i]
                        del dpa[i]

                if len(smatchin) < 2: continue

                #Finds mode of difference position angle but allowing values to be +/- patolerance
                dpamode = astrometrystats.most(dpa,
                                               vmin=patolerance * 3,
                                               vmax=patolerance * 3)

                #Remove deviant matches by PA
                for i in range(len(smatchin) - 1, -1, -1):
                    if abs(dpa[i] - dpamode) > patolerance:
                        del smatchin[i]
                        del cmatchin[i]
                        del dpa[i]

                if len(smatchin) < 2: continue

                #Finds the number of degenerate matches
                ndegeneracies = len(smatchin) - len(
                    astrometrystats.unique(smatchin)) + len(cmatchin) - len(
                        astrometrystats.unique(cmatchin))
                # this isn't quite accurate (overestimates if degeneracies are mixed up)

                #Save values
                mpa.append(dpamode)
                primarymatchs.append(si)
                primarymatchc.append(ci)
                smatch.append(smatchin)
                cmatch.append(cmatchin)
                nmatch.append(len(smatchin) - ndegeneracies)

                #If the number of unique sextractor sources is greater than 6, count as a great match
                if (len(smatchin) - ndegeneracies > 6): countgreatmatches += 1

        #Breaks loop if over 16 great matches are found and fastmatch
        if countgreatmatches > 16 and fastmatch == 1:
            break  #save processing time

    #If no matches found end program and return empty lists
    nmatches = len(smatch)
    if (nmatches == 0):
        print 'Found no potential matches of any sort (including pairs).'
        print 'The algorithm is probably not finding enough real stars to solve the field.  Check seeing.'
        return [], [], []

    #Get rid of matches that don't pass the reqmatch cut (2nd cut after removing bad position angles)
    for i in range(len(primarymatchs) - 1, -1, -1):
        if nmatch[i] < reqmatch:
            del mpa[i]
            del primarymatchs[i]
            del primarymatchc[i]
            del smatch[i]
            del cmatch[i]
            del nmatch[i]

#If no remaining matches then exit program
    if len(smatch) < 1:
        print 'Found no matching clusters of reqmatch =', reqmatch
        return [], [], []

    #If we still have lots of matches, get rid of those with the minimum number of submatches
    #(that is, increase reqmatch by 1)
    minmatch = min(nmatch)
    countnotmin = 0

    #Finds how many matches have more than the minimum require matches
    for n in nmatch:
        if n > minmatch: countnotmin += 1

    #If the number of matches is above 16 and there are more than 3 sources with more than the
    #required number of matches, then delete any source with the bare minimum number of matches
    if len(nmatch) > 16 and countnotmin > 3:
        print 'Too many matches: increasing reqmatch to', reqmatch + 1
        for i in range(len(primarymatchs) - 1, -1, -1):
            if nmatch[i] == minmatch:
                del mpa[i]
                del primarymatchs[i]
                del primarymatchc[i]
                del smatch[i]
                del cmatch[i]
                del nmatch[i]

    nmatches = len(
        smatch
    )  # recalculate with the new reqmatch and with prunes supposedly removed
    print 'Found', nmatches, 'candidate matches.'

    # Kill the bad matches
    rejects = 0

    # Use only matches with a consistent PA (finds mode counting those within 3 patolerance)
    offpa = astrometrystats.most(mpa,
                                 vmin=3 * patolerance,
                                 vmax=3 * patolerance)

    #Removes values with rotational positional offsets that are above tolerance, then removes
    #values that are above 2 sigma of those values
    if len(smatch) > 2:

        #Coarse iteration for anything away from the mode
        for i in range(len(primarymatchs) - 1, -1, -1):
            if abs(mpa[i] - offpa) > patolerance:
                del mpa[i]
                del primarymatchs[i]
                del primarymatchc[i]
                del smatch[i]
                del cmatch[i]
                del nmatch[i]
                rejects += 1

        medpa = astrometrystats.median(mpa)
        stdevpa = astrometrystats.stdev(mpa)
        refinedtolerance = (
            2.0 * stdevpa
        )  #VLT Changed from arbitrary value of 2.2 from original script

        #Fine iteration to flag outliers now that we know most are reliable
        for i in range(len(primarymatchs) - 1, -1, -1):
            if abs(mpa[i] - offpa) > refinedtolerance:
                del mpa[i]
                del primarymatchs[i]
                del primarymatchc[i]
                del smatch[i]
                del cmatch[i]
                del nmatch[i]
                rejects += 1  #these aren't necessarily bad, just making more manageable.

    # New verification step: calculate distances and PAs between central stars of matches
    ndistflags = [0] * len(primarymatchs)
    for v in range(2):  #two iterations

        if len(primarymatchs) == 0: break

        #Find distances between central stars of matches and compare sextractor source distances to catalog sources
        for i in range(len(primarymatchs)):
            for j in range(len(primarymatchs)):
                if i == j: continue
                si = primarymatchs[i]
                ci = primarymatchc[i]
                sj = primarymatchs[j]
                cj = primarymatchc[j]

                sexdistij = distance(sexlist[si], sexlist[sj])
                catdistij = distance(catlist[ci], catlist[cj])

                try:
                    if abs((sexdistij / catdistij) - 1.0) > 0.03:
                        ndistflags[i] += 1
                except:  # (occasionally will get divide by zero)
                    pass

        #Delete bad clusters that were flagged for every match
        ntestmatches = len(primarymatchs)
        for i in range(ntestmatches - 1, -1, -1):
            if ndistflags[
                    i] == ntestmatches - 1:  #if every comparison is bad, this is a bad match
                del mpa[i]
                del primarymatchs[i]
                del primarymatchc[i]
                del smatch[i]
                del cmatch[i]
                del nmatch[i]
                rejects += 1

    print 'Rejected', rejects, 'bad matches.'
    nmatches = len(primarymatchs)
    print 'Found', nmatches, 'good matches.'

    #If no remaining matches, return empty lists
    if nmatches == 0:
        return [], [], []

    #Returns pixel scale (great circle distance in catalog [ra, dec]/cartesian distance in sextractor source [x,y])
    pixscalelist = []
    if len(primarymatchs) >= 2:
        for i in range(len(primarymatchs) - 1):
            for j in range(i + 1, len(primarymatchs)):
                si = primarymatchs[i]
                ci = primarymatchc[i]
                sj = primarymatchs[j]
                cj = primarymatchc[j]
                try:
                    pixscalelist.append(
                        distance(catlist[ci], catlist[cj]) /
                        imdistance(sexlist[si], sexlist[sj]))
                except:
                    pass
        pixelscale = astrometrystats.median(pixscalelist)
        pixelscalestd = astrometrystats.stdev(pixscalelist)

        if len(primarymatchs) >= 3:
            print 'Refined pixel scale measurement: %.4f"/pix (+/- %.4f)' % (
                pixelscale, pixelscalestd)
        else:
            print 'Refined pixel scale measurement: %.4f"/pix' % pixelscale

#If showmatches keyword set then print which objects match
    for i in range(len(primarymatchs)):
        si = primarymatchs[i]
        ci = primarymatchc[i]
        print '%3i' % si, 'matches', '%3i' % ci, ' (dPA =%7.3f)' % mpa[i],

        #Keyword set in main program
        if showmatches:
            print
            if len(smatch[i]) < 16:
                print '  ', si, '-->', smatch[i],
                if len(smatch[i]) >= 7: print
                print '  ', ci, '-->', cmatch[i]
            else:
                print '  ', si, '-->', smatch[i][0:10], '+', len(
                    smatch[i]) - 10, 'more'
                print '  ', ci, '-->', cmatch[i][
                    0:10], '+'  #, len(cmatch[i])-10, ' more'
            if i + 1 >= 10 and len(primarymatchs) - 10 > 0:
                print(len(primarymatchs) - 10), 'additional matches not shown.'
                break
        else:
            print ':', str(len(smatch[i])).strip(), 'rays'

#Create region files for DS9 with the sextractor sources (matchlines.im.reg) and catalog (matchlines.wcs.reg)
    out = open('matchlines.im.reg', 'w')
    i = -1
    color = 'red'
    out.write(
        '# Region file format: DS9 version 4.0\nglobal color=' + color +
        ' font="helvetica 10 normal" select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\n'
    )
    out.write('image\n')
    for i in range(len(primarymatchs)):
        si = primarymatchs[i]
        for j in range(len(smatch[i])):
            sj = smatch[i][j]
            out.write(
                "line(%.3f,%.3f,%.3f,%.3f) # line=0 0\n" %
                (sexlist[si].x, sexlist[si].y, sexlist[sj].x, sexlist[sj].y))
    out.close()

    out = open('matchlines.wcs.reg', 'w')
    i = -1
    color = 'green'
    out.write(
        '# Region file format: DS9 version 4.0\nglobal color=' + color +
        ' font="helvetica 10 normal" select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\n'
    )
    out.write('fk5\n')
    for i in range(len(primarymatchs)):
        ci = primarymatchc[i]
        for j in range(len(smatch[i])):
            cj = cmatch[i][j]
            out.write("line(%.5f,%.5f,%.5f,%.5f) # line=0 0\n" %
                      (catlist[ci].ra, catlist[ci].dec, catlist[cj].ra,
                       catlist[cj].dec))
    out.close()

    #future project: if not enough, go to the secondary offsets

    #Returns sextractor sources and catalog sources that appear to have matches along with the mode of the position angle
    return (primarymatchs, primarymatchc, mpa)
def autoastrometry(filename,pixelscale=-1,pa=-999,inv=0,uncpa=-1,userra=-999, userdec=-999, minfwhm=1.5,maxfwhm=20,maxellip=0.5,boxsize=-1,maxrad=-1,tolerance=0.010,catalog='',nosolve=0,overwrite=False, outfile='', saturation=-1, quiet=False):
  
    # Get some basic info from the header  
    try: 
       fits = pyfits.open(filename)
       fits.verify('silentfix')
    except:
       print 'Error opening', filename
       if os.path.isfile(filename)==False: print 'File does not exist.'
       return -1
    h = fits[0].header  #ideally check for primary extension, or even iterate
    sfilename = filename
    
    #Position angle set to 0 if pixel scale set
    if pixelscale > 0 and pa == -999: pa = 0

	#If pixel scale set and position angle set (default is -999), 
	#calculate CD*, CR* header keywords and write as temp.fits
    if pixelscale > 0 and pa > -360:
 
       parad = pa * numpy.pi / 180.
       pxscaledeg = pixelscale / 3600.
       
       if inv > 0: 
          parity = -1
       else:
          parity = 1
          
       if  360. > userra >= 0.:
          ra = userra
       else:
          try:
             ra = astrometrystats.rasex2deg(h['RA'])
          except:
             ra = astrometrystats.rasex2deg(h['CRVAL1'])
             
       if 360. > userdec >= 0.:
          dec = userdec
       else:
          try:
             dec = astrometrystats.decsex2deg(h['DEC'])
          except:
             dec = astrometrystats.decsex2deg(h['CRVAL2'])
       
       epoch = float(h.get('EPOCH', 2000))
       equinox = float(h.get('EQUINOX', epoch)) #If RA and DEC are not J2000 then convert

       if abs(equinox-2000) > 0.5:
           print 'Converting equinox from', equinox, 'to J2000'
           try:
              j2000 = ephem.Equatorial(ephem.Equatorial(str(ra/15), str(dec), epoch=str(equinox)),epoch=ephem.J2000)
              [ra, dec] = [astrometrystats.rasex2deg(j2000.ra), astrometrystats.decsex2deg(j2000.dec)]
           except:
              print 'PyEphem is not installed but is required to precess this image.'
              return -1
       h.update("CD1_1",  pxscaledeg * numpy.cos(parad)*parity)
       h.update("CD1_2",  pxscaledeg * numpy.sin(parad))
       h.update("CD2_1", -pxscaledeg * numpy.sin(parad)*parity)
       h.update("CD2_2",  pxscaledeg * numpy.cos(parad))
       h.update("CRPIX1", h['NAXIS1']/2)
       h.update("CRPIX2", h['NAXIS2']/2)
       h.update("CRVAL1", ra)
       h.update("CRVAL2", dec)
       h.update("CTYPE1","RA---TAN")
       h.update("CTYPE2","DEC--TAN")
       h.update("EQUINOX",2000.0)
       
       if os.path.isfile('temp.fits'): os.remove('temp.fits')
       fits[0].header = h
       fits.writeto('temp.fits',output_verify='silentfix') #,clobber=True
       fits.close()
       fits = pyfits.open('temp.fits')
       h = fits[0].header
       sfilename = 'temp.fits'

    #Read the WCS values from file header (even if we put it there in the first place)
    try:
        # no longer drawing RA and DEC from here.
        nxpix = h['NAXIS1']
        nypix = h['NAXIS2']
    except:
        print 'Cannot find necessary WCS header keyword NAXIS*'
        sys.exit(1)
    try: 
        cra  =  float(h['CRVAL1'])
        cdec = float(h['CRVAL2'])
        crpix1 = float(h['CRPIX1'])  
        crpix2 = float(h['CRPIX2'])
        cd11 = float(h['CD1_1'])
        cd22 = float(h['CD2_2'])
        cd12 = float(h['CD1_2']) # deg / pix
        cd21 = float(h['CD2_1'])
    except:
        print 'Cannot find necessary WCS header keyword CRVAL*, CRPIX*, or CD*_*'
        print 'Must specify pixel scale (-px VAL) or provide provisional basic WCS info via CD matrix.'
        sys.exit(1)

	#This section deals with manipulating WCS coordinates, for thorough description see:
	#iraf.noao.edu/iraf/ftp/misc/fitswcs_draft.ps

	#Determine parity from sign of determinant of CD matrix   
    if cd11 * cd22 < 0 or cd12 * cd21 > 0:
       parity = -1
    else:
       parity = 1
        
    #Calculates CDELTA1 (xscale) and CDELTA2 (yscale) which is how much RA or DEC 
    #changes when you move along a column or row
    xscale = numpy.sqrt(cd11**2 + cd21**2)
    yscale = numpy.sqrt(cd12**2 + cd22**2)
    
    #Calculates CROTA2 based from transformations between CD matrix and CDELT values
    initpa = -parity * numpy.arctan2(cd21 * yscale, cd22 * xscale) * 180 / numpy.pi
    
    #Find field width based on largest dimension and calculates the area of the field
    #as well as the pixel scale in arcseconds
    xscale = abs(xscale)
    yscale = abs(yscale)
    fieldwidth = max(xscale * nxpix, yscale * nypix) * 3600.    
    area_sqdeg = xscale * nxpix * yscale * nypix
    area_sqmin = area_sqdeg * 3600. 
    area_sqsec = area_sqmin * 3600. 
    pixscale = numpy.sqrt(xscale*yscale) * 3600.

	#Finds center pixel in each dimension
    centerx = nxpix/2
    centery = nypix/2
    
    #Calculate how the center pixel relates the to the header value's crpix1/2.  
    #Theoretically, centerx and crpix1 should be the same.  But most of the time, they are not. 
    #This is due to a number of reasons, ranging from the algorithm used to calculate the 
    #astrometry to simply cropping the image at some point in the reduction.  
    #So, the crpix header value may be hundreds of pixels off from the actual center of the actual field.
    centerdx = centerx - crpix1
    centerdy = centery - crpix2
    
    #Calculate the RA and DEC at center of field to correct initial guess
    centerra  = cra  - centerdx*xscale*numpy.cos(initpa*numpy.pi/180.) + centerdy*yscale*numpy.sin(initpa*numpy.pi/180.)
    centerdec = cdec + parity*centerdx*xscale*numpy.sin(-initpa*numpy.pi/180.) + centerdy*yscale*numpy.cos(initpa*numpy.pi/180.)
    print 'cra=%10.6f, centerdx=%10.6f, xscale=%10.6f, centerdy=%10.6f, yscale=%10.6f' % (cra,centerdx,xscale,centerdy,yscale)
    # this has only been checked for a PA of zero.
  
    if quiet == False:
       print 'Initial WCS info:'
       print '   pixel scale:     x=%.4f"/pix,   y=%.4f"/pix' % (xscale*3600, yscale*3600)
       print '   position angle: PA=%.2f' % initpa
       if parity ==  1: print '   normal parity'
       if parity == -1: print '   inverse parity'
       print '   center:        RA=%10.6f, dec=%9.6f' % (centerra, centerdec)
       print '   field width: %10.6f' % (fieldwidth)

	#Run sextract (runs sextractor) to produce image star catalog
    goodsexlist = astrometrysources.sextract(sfilename, nxpix, nypix, 3, 12, minfwhm=minfwhm, maxfwhm=maxfwhm, maxellip=maxellip, saturation=saturation, sexpath=sexpath)
    
    #If there are less than 4 good objects, ends program and writes images to txt and region files
    ngood = len(goodsexlist)
    if ngood < 4:
       print 'Only', ngood, 'good stars were found in the image.  The image is too small or shallow, the detection'
       print 'threshold is set too high, or stars and cosmic rays are being confused.'      
       #Saves text file that contains RA, DEC, and mag of sextractor list
       writetextfile('det.init.txt', goodsexlist)
       writeregionfile('det.im.reg', goodsexlist, 'red', 'img')
       return -1

	#Finds source number density
    density = len(goodsexlist) / area_sqmin
    print 'Source density of %f4 /arcmin^2' % density
    
    #If set to only solve for catalog and not astrometry, save good list
    if nosolve == 1: 
       if catalog == '': catalog = 'det.ref.txt'
       #Saves text file that contains RA, DEC, and mag of sextractor list
       writetextfile(catalog, goodsexlist)
       return

    #If no catalog specified, find catalog with > 15 entries for center RA and DEC encircled by radius of 90 arcseconds
    #If no catalog found after that, end program
    if catalog == '':
    
        trycats = ['tmpsc','sdss', 'ub2', 'tmc']
        for trycat in trycats:
            testqueryurl = "http://tdc-www.harvard.edu/cgi-bin/scat?catalog=" + trycat +  "&ra=" + str(centerra) + "&dec=" + str(centerdec) + "&system=J2000&rad=" + str(-90)
            check = urllib.urlopen(testqueryurl)
            checklines = check.readlines()
            check.close()
            if len(checklines) > 15:
                catalog = trycat
                print 'Using catalog', catalog
                break
        if (catalog == ''):
            print 'No catalog is available.  Check your internet connection.'
            return -1
	
    #Load in reference star catalog with boxsize
    if (boxsize == -1):
        boxsize = fieldwidth

    catlist = astrometrysources.getcatalog(catalog, centerra, centerdec, boxsize)
    
    ncat = len(catlist)
    catdensity = ncat / (2*boxsize/60.)**2
    print ncat, 'good catalog objects.'
    print 'Source density of %f4 /arcmin^2' % catdensity
    
    #Throws up warning if very few catalog objects, stops program if no catalog objects found
    if 0 < ncat < 5:
       print 'Only ', ncat, ' catalog objects in the search zone.  Increase the magnitude threshold or box size.'

    if ncat == 0 :
       print
       print 'No objects found in catalog.'
       print 'The web query failed, all stars were excluded by the FHWM clip, or the image'
       print 'is too small.  Check input parameters or your internet connection.'
       return -1   
  
    #If this image is actually shallower than reference catalog, trim the reference catalog down
    if ncat > 16 and catdensity > 3 * density:
        print 'Image is shallow.  Trimming reference catalog...'
        while catdensity > 3 * density:
            catlist = catlist[0:len(catlist)*4/5]
            ncat = len(catlist)
            catdensity = ncat / (2*boxsize/60.)**2
        
    #If the image is way deeper than USNO, trim the image catalog down
    if ngood > 8 and density > 4 * catdensity:
        print 'Image is deep.  Trimming image catalog...'
        while density > 4 * catdensity and ngood > 8:
            goodsexlist = goodsexlist[0:len(goodsexlist)*4/5]
            ngood = len(goodsexlist)
            density = ngood / area_sqmin

    #If too many objects, do some more trimming
    if ngood*ncat > 120*120*4:
        print 'Image and/or catalog still too deep.  Trimming...'
        while ngood*ncat > 120*120*4:
            if density > catdensity: 
                goodsexlist = goodsexlist[0:len(goodsexlist)*4/5]
                ngood = len(goodsexlist)
                density = ngood / area_sqmin
            else:
                catlist = catlist[0:len(catlist)*4/5]
                ncat = len(catlist)
                catdensity = ncat / (2*boxsize/60.)**2   
    
    #Remove fainter object in close pairs for both lists
    goodsexlist = astrometrydist.tooclose(goodsexlist, minsep=3)
    catlist = astrometrydist.tooclose(catlist, minsep=3)
    
    #Saves text file that contains RA, DEC, and mag of sextractor list
    writetextfile('det.init.txt', goodsexlist)
    writeregionfile('det.im.reg', goodsexlist, 'red', 'img')
    writetextfile('cat.txt', catlist)
    writeregionfile('cat.wcs.reg', catlist, 'green', 'wcs')   

    ##### The catalogs have now been completed. Now start getting into the actual astrometry. #####
    
    #Maximum radius (in arcseconds) calculated by looking at the radius of 15 object in the sparsest dataset
    #Must at least 60 arcseconds or 75% of the field width (whichever is smaller)
    minrad = 5.0
    if (maxrad == -1):  
    	#60 arcsec/arcmin *sqrt(15 stars/(stars/arcsec^2)) / 2 <-- density is for a box, so half the length of the box is the radius                                          
        maxrad = 30.0*(15.0 /min(density,catdensity))**0.5  
        maxrad = max(maxrad, 60.0)
        
        if maxrad == 60.0:                           
             minrad = 10.0   # in theory could scale this up further to reduce #comparisons i.e. instead of 15 stars, x stars
        maxrad = min(maxrad, fieldwidth*3./4)       
        
    #Finds the number of objects expected within circular area (chooses smaller of the entire field or of area 
    #in between min and max radius). NOTE: density is per arcmin^2, while the radii are in arcsec, hence the conversion factor. 
    circlearea     = (numpy.pi*(maxrad/60.)**2 - numpy.pi*(minrad/60)**2) #in arcmin^2
    circdensity    = density * min([area_sqmin, circlearea])
    circcatdensity = catdensity * circlearea	#Finds number of catalog objects expected within circular area
    catperimage    = catdensity * area_sqmin	#Finds number of catalog objects expected within field

    print 'After trimming: '
    print '   ', len(goodsexlist), 'detected objects (%.2f/arcmin^2, %.1f/searchzone)' % (density, circdensity)
    print '   ', len(catlist),     'catalog objects (%.2f/arcmin^2, %.1f/searchzone)' % (catdensity, circcatdensity)
	
	#Sets position angle tolerance and calculates the expected number of false multiples
    patolerance = defaultpatolerance
    expectfalsepairs = ngood * ncat * circdensity**1 * circcatdensity**1 * tolerance**1 * (patolerance/360.)**0
    expectfalsetrios = ngood * ncat * circdensity**2 * circcatdensity**2 * tolerance**2 * (patolerance/360.)**1
    expectfalsequads = ngood * ncat * circdensity**3 * circcatdensity**3 * tolerance**3 * (patolerance/360.)**2
    expectfalsequint = ngood * ncat * circdensity**4 * circcatdensity**4 * tolerance**4 * (patolerance/360.)**3

	#Guess that 30% of the sextractor sources overlap with stars, finds estimate of how many real matches we expect
    overlap1 = 0.3 * min(1,catdensity/density) # fraction of stars in image that are also in catalog - a guess
    truematchesperstar = (circdensity * overlap1) # but how many matches >3 and >4?  some annoying binomial thing
    
    #Default required match is 3 (triangle), but can require more if we expect a log of false triples or less if not many sources in either image
    reqmatch = 3
    if expectfalsetrios > 30 and truematchesperstar >= 4: reqmatch = 4   
       #should check that this will actually work for the catalog, too.
    if catperimage <= 6 or ngood <= 6: reqmatch = 2 
    if catperimage <= 3 or ngood <= 3: reqmatch = 1
        #for an extremely small or shallow image

	#Calculates the matched stars between sextractor and catalog
    print 'Pair comparison search radius: %.2f"'%maxrad
    print 'Using reqmatch =', reqmatch
    (primarymatchs, primarymatchc, mpa) = astrometrydist.distmatch(goodsexlist, catlist, maxrad, minrad, reqmatch, patolerance, uncpa, showmatches=showmatches, fastmatch=fastmatch)
    
    #Quits program if no matches or too few matches found (gives different error readouts)
    nmatch = len(primarymatchs)
    if nmatch == 0:
        print ' No valid matches found!'
        if quiet == False:
           print ' Possible issues:'
           print '  - The specified pixel scale (or PA or parity) is incorrect.  Double-check the input value.'
           print '  - The field is outside the catalog search region.  Check header RA/DEC or increase search radius.'
           print '  - The routine is flooded by bad sources.  Specify or check the input seeing.'
           print '  - The routine is flagging many real stars.  Check the input seeing.'
           print ' You can display a list of detected/catalog sources using det.im.reg and cat.wcs.reg.'
        return -1
    if nmatch <= 2:
        print 'Warning: only', nmatch, 'match(es).  Astrometry may be unreliable.'
        if quiet == False:
           print '   Check the pixel scale and parity and consider re-running.'
        return -1
        warning = 1

    #We now have the PA and a list of stars that are almost certain matches.
    offpa = astrometrystats.median(mpa)  #get average PA from the excellent values
    stdevpa = astrometrystats.stdev(mpa)
    
    skyoffpa = -parity*offpa # This appears to be necessary for the printed value to agree with our normal definition.

    print 'PA offset:'
    print '  dPA = %.3f  (unc. %.3f)' % (skyoffpa, stdevpa)

    # Rotate the image to the new, correct PA
    #  NOTE: when CRPIX don't match CRVAL this shifts the center and screws things up.  
    #  I don't understand why they don't always match.  [[I think this was an equinox issue.
    #  should be solved now, but be alert for further problems.]]
    
    #Greisen et al.:
    #WCS_i = SUM[j] (CD_ij)(p_j - CRPIX_j)      i.e.
    # RA - CRVAL1 = CD1_1 (x - CRPIX1) + CD1_2 (y - CRPIX2)
    #dec - CRVAL2 = CD2_1 (x - CRPIX1) + CD2_2 (y - CRPIX2)   [times a projection scale...]

    #Rotate CD matrix to account for additional rotation calculated from catalog star matching
    rot = offpa * numpy.pi/180
      #...the image itself
    h.update("CD1_1", numpy.cos(rot)*cd11 - numpy.sin(rot)*cd21 )
    h.update("CD1_2", numpy.cos(rot)*cd12 - numpy.sin(rot)*cd22 )  # a parity issue may be involved here?
    h.update("CD2_1", numpy.sin(rot)*cd11 + numpy.cos(rot)*cd21 )
    h.update("CD2_2", numpy.sin(rot)*cd12 + numpy.cos(rot)*cd22 )
      #...the coordinates (so we don't have to resex) 
    for i in range(len(goodsexlist)):  #do all of them, though this is not necessary
        goodsexlist[i].rotate(offpa,cra,cdec)

    #Saves text file that contains RA, DEC, and mag of sextractor list
    writetextfile('det.wcs.txt', goodsexlist)
    
    #Calculate shift in RA and DEC for each object compared to its catalog match
    imraoffset = []
    imdecoffset = []
    for i in range(len(primarymatchs)):
        imraoffset.append(goodsexlist[primarymatchs[i]].ra - catlist[primarymatchc[i]].ra)
        imdecoffset.append(goodsexlist[primarymatchs[i]].dec - catlist[primarymatchc[i]].dec)
    
    #Find the median and standard deviation of offsets    
    raoffset = -astrometrystats.median(imraoffset)
    decoffset = -astrometrystats.median(imdecoffset)
    rastd = astrometrystats.stdev(imraoffset)*numpy.cos(cdec*numpy.pi/180)  # all of these are in degrees
    decstd = astrometrystats.stdev(imdecoffset)
    stdoffset = numpy.sqrt(rastd**2 + decstd**2)
    
    #Change from degrees to arcseconds
    raoffsetarcsec = raoffset*3600*numpy.cos(cdec*numpy.pi/180)
    decoffsetarcsec = decoffset*3600
    totoffsetarcsec = (raoffsetarcsec**2 + decoffset**2)**0.5
    stdoffsetarcsec = stdoffset*3600
    
    print 'Spatial offset:'
    print '  dra = %.2f",  ddec = %.2f"  (unc. %.3f")' % (raoffsetarcsec, decoffsetarcsec, stdoffsetarcsec)
    
    #If standard deviation of total offset is larger than 10 arcseconds end program
    warning = 0
    if (stdoffset*3600 > 10.0):
        print 'WARNING: poor solution - some matches may be bad.  Check pixel scale?'
        return -1
        warning = 1
    
    #Shift center pixel to match catalog values
    h.update("CRVAL1", cra + raoffset)
    h.update("CRVAL2", cdec + decoffset)
    
    #Add keywords to header to detail changes made to WCS coordinates
    try:
       oldcat = h['ASTR_CAT']
       h.update("OLD_CAT",oldcat, "Earlier reference catalog")
    except:
       pass
    h.update("ASTR_CAT", catalog, "Reference catalog for vlt_autoastrometry")
    h.update("ASTR_UNC", stdoffsetarcsec, "Astrometric scatter vs. catalog (arcsec)")
    h.update("ASTR_SPA", stdevpa, "Measured uncertainty in PA (degrees)")
    h.update("ASTR_DPA", skyoffpa, "Change in PA (degrees)")
    h.update("ASTR_OFF", totoffsetarcsec, "Change in center position (arcsec)")
    h.update("ASTR_NUM", len(primarymatchs), "Number of matches")

    #Write out a match list to allow doing a formal fit with WCStools.
    outmatch = open('match.list','w')
    for i in range(len(primarymatchs)):
        si = primarymatchs[i]
        ci = primarymatchc[i]
        outmatch.write("%s %s  %s %s\n" % (goodsexlist[si].x, goodsexlist[si].y, catlist[ci].ra, catlist[ci].dec))
    outmatch.close()                                                     
    
    # Could repeat with scale adjustment
    # Could then go back to full good catalog and match all sources
    
    #Create new file with new header
    if overwrite: outfile = filename
    if outfile == '': 
        slashpos = filename.rfind('/')
        dir = filename[0:slashpos+1]
        fil = filename[slashpos+1:]
        outfile = dir+'a'+fil # alternate behavior would always output to current directory
    try:
        os.remove(outfile)
    except:
        pass
    fits[0].header = h
    fits.writeto(outfile,output_verify='silentfix') #,clobber=True
    print 'Written to '+outfile

    fits.close()
    
    #Return relevant offsets
    return (nmatch, skyoffpa, stdevpa, raoffsetarcsec, decoffsetarcsec, stdoffsetarcsec)
Example #3
0
def distmatch(sexlist, catlist, maxrad=180, minrad=10, reqmatch=3, patolerance=1.2,uncpa=-1, showmatches=0, fastmatch=1):
    
    if reqmatch < 2:
       print 'Warning: reqmatch >=3 suggested'
    if patolerance <= 0: 
       print 'PA tolerance cannot be negative!!!'
       patolerance = abs(patolerance)
    if uncpa < 0: uncpa = 720

    declist = []
    for s in sexlist:
       declist.append(s.dec_rad)
    avdec_rad = astrometrystats.median(declist)       		# faster distance computation
    rascale = numpy.cos(avdec_rad)          # will mess up meridian crossings, however

 	#Calculates distances between objects in same list for both image catalog and reference catalog  
    (sexdists, sexmatchids) = calcdist(sexlist, maxrad, minrad, rascale)
    (catdists, catmatchids) = calcdist(catlist, maxrad, minrad, rascale)
    
    # Now look for matches in the reference catalog to distances in the image catalog.    
    countgreatmatches = 0

    smatch = []
    cmatch = []
    mpa = []
    offset = []
    offpa = []
    nmatch = []   
    primarymatchs = []
    primarymatchc = []
    
    #For each sextractor source A
    for si in range(len(sexdists)):
        sexdistarr = sexdists[si]
        sexidarr = sexmatchids[si]
        if len(sexdistarr) < 2: continue
        
        #For each catalog source B
        for ci in range(len(catdists)):
            catdistarr = catdists[ci]
            catidarr = catmatchids[ci]
            if len(catdistarr) < 2: continue
            match = 0
            smatchin = []
            cmatchin = []
            
            #For each sextractor source A, use the distances to other sextractor sources
            #that are within the radius range
            for sj in range(len(sexdistarr)):
                sexdist = sexdistarr[sj]
                newmatch = 1
                
                #For each catalog source B, use the distances to other catalog sources
                #that are within the radius range and compare the ratio to 
                #(sextractor source A - sextractor sources) to (catalog source B - catalog sources)
                #If within 3% then save the match (only count 1 match per catalog row)
                for cj in range(len(catdistarr)):
                    catdist = catdistarr[cj]
                    if abs((sexdist/catdist)-1.0) < 0.03:

                        match += newmatch
                        newmatch = 0 #further matches before the next sj loop indicate degeneracies
                        smatchin.append(sexmatchids[si][sj])
                        cmatchin.append(catmatchids[ci][cj])                 

			#If number of matches is above or equal to required number of matches for sextractor source A
			#(i.e. sextractor source A's distance to other sources makes it a likely candidate to be a catalog source)
			#Find difference in position angle between matched angle between sextractor source A and matched source with
			#catalog source B and matched catalog source.
			#Remove values that are larger than user specified value or if above position angle tolerance.
            if match >= reqmatch: 
                
                dpa = []
                
                # Here, dpa[n] is the mean rotation of the PA from the primary star of this match
                #  to the stars in its match RELATIVE TO those same angles for those same stars
                #  in the catalog.  Therefore it is a robust measurement of the rotation.
                for i in range(len(smatchin)):
                    ddpa = posangle(sexlist[si],sexlist[smatchin[i]]) - posangle(catlist[ci],catlist[cmatchin[i]])
                    while ddpa > 200: ddpa  -= 360.
                    while ddpa < -160: ddpa += 360.
                    dpa.append(ddpa)

                #If user was confident the initial PA was right, remove bad PA's right away
                for i in range(len(smatchin)-1,-1,-1):
                    if abs(dpa[i]) > uncpa: 
                        del smatchin[i]
                        del cmatchin[i]
                        del dpa[i]
                    
                if len(smatchin) < 2: continue
                
                #Finds mode of difference position angle but allowing values to be +/- patolerance    
                dpamode = astrometrystats.most(dpa, vmin=patolerance*3, vmax=patolerance*3)
                
                #Remove deviant matches by PA
                for i in range(len(smatchin)-1,-1,-1):
                    if abs(dpa[i] - dpamode) > patolerance:
                        del smatchin[i]
                        del cmatchin[i]
                        del dpa[i]
                				
                if len(smatchin) < 2: continue
                
                #Finds the number of degenerate matches
                ndegeneracies = len(smatchin)-len(astrometrystats.unique(smatchin)) + len(cmatchin)-len(astrometrystats.unique(cmatchin))
                    # this isn't quite accurate (overestimates if degeneracies are mixed up)

				#Save values
                mpa.append(dpamode)
                primarymatchs.append(si)
                primarymatchc.append(ci)
                smatch.append(smatchin)
                cmatch.append(cmatchin)
                nmatch.append(len(smatchin)-ndegeneracies)
                
                #If the number of unique sextractor sources is greater than 6, count as a great match
                if (len(smatchin)-ndegeneracies > 6): countgreatmatches += 1
        
        #Breaks loop if over 16 great matches are found and fastmatch
        if countgreatmatches > 16 and fastmatch == 1: break #save processing time
    
    #If no matches found end program and return empty lists
    nmatches = len(smatch)
    if (nmatches == 0):
        print 'Found no potential matches of any sort (including pairs).'
        print 'The algorithm is probably not finding enough real stars to solve the field.  Check seeing.'
        return [], [], []   
    
    #Get rid of matches that don't pass the reqmatch cut (2nd cut after removing bad position angles)
    for i in range(len(primarymatchs)-1,-1,-1):
        if nmatch[i] < reqmatch:
            del mpa[i]
            del primarymatchs[i]
            del primarymatchc[i]
            del smatch[i]
            del cmatch[i]
            del nmatch[i]

	#If no remaining matches then exit program
    if len(smatch) < 1:
        print 'Found no matching clusters of reqmatch =', reqmatch
        return [], [], []

    #If we still have lots of matches, get rid of those with the minimum number of submatches
    #(that is, increase reqmatch by 1)
    minmatch = min(nmatch)
    countnotmin = 0
    
    #Finds how many matches have more than the minimum require matches
    for n in nmatch:
       if n > minmatch: countnotmin += 1
    
    #If the number of matches is above 16 and there are more than 3 sources with more than the 
    #required number of matches, then delete any source with the bare minimum number of matches
    if len(nmatch) > 16 and countnotmin > 3:
        print 'Too many matches: increasing reqmatch to', reqmatch+1
        for i in range(len(primarymatchs)-1,-1,-1):
            if nmatch[i] == minmatch:
                del mpa[i]
                del primarymatchs[i]
                del primarymatchc[i]
                del smatch[i]
                del cmatch[i]
                del nmatch[i]
          
    nmatches = len(smatch) # recalculate with the new reqmatch and with prunes supposedly removed
    print 'Found',nmatches,'candidate matches.'

    # Kill the bad matches
    rejects = 0
    
    # Use only matches with a consistent PA (finds mode counting those within 3 patolerance)
    offpa = astrometrystats.most(mpa, vmin=3*patolerance, vmax=3*patolerance)
    
    #Removes values with rotational positional offsets that are above tolerance, then removes
    #values that are above 2 sigma of those values
    if len(smatch) > 2:    

        #Coarse iteration for anything away from the mode
        for i in range(len(primarymatchs)-1,-1,-1):
            if abs(mpa[i] - offpa) > patolerance:
                del mpa[i]
                del primarymatchs[i]
                del primarymatchc[i]
                del smatch[i]
                del cmatch[i]
                del nmatch[i]
                rejects += 1
    
        medpa = astrometrystats.median(mpa)
        stdevpa = astrometrystats.stdev(mpa)
        refinedtolerance = (2.0 * stdevpa) #VLT Changed from arbitrary value of 2.2 from original script
        
        #Fine iteration to flag outliers now that we know most are reliable
        for i in range(len(primarymatchs)-1,-1,-1):
            if abs(mpa[i] - offpa) > refinedtolerance:
                del mpa[i]
                del primarymatchs[i]
                del primarymatchc[i]
                del smatch[i]
                del cmatch[i]
                del nmatch[i]
                rejects += 1  #these aren't necessarily bad, just making more manageable.
    	
    # New verification step: calculate distances and PAs between central stars of matches
    ndistflags = [0]*len(primarymatchs)
    for v in range(2):  #two iterations

        if len(primarymatchs) == 0: break

        #Find distances between central stars of matches and compare sextractor source distances to catalog sources
        for i in range(len(primarymatchs)):
            for j in range(len(primarymatchs)):
                if i == j: continue
                si = primarymatchs[i]
                ci = primarymatchc[i]
                sj = primarymatchs[j]
                cj = primarymatchc[j]
    
                sexdistij = distance(sexlist[si], sexlist[sj])
                catdistij = distance(catlist[ci], catlist[cj])
                
                try:
                   if abs((sexdistij/catdistij)-1.0) > 0.03:
                      ndistflags[i] += 1
                except:  # (occasionally will get divide by zero)
                   pass

        #Delete bad clusters that were flagged for every match
        ntestmatches = len(primarymatchs)
        for i in range(ntestmatches-1,-1,-1):
            if ndistflags[i] == ntestmatches-1:   #if every comparison is bad, this is a bad match
                del mpa[i]
                del primarymatchs[i]
                del primarymatchc[i]
                del smatch[i]
                del cmatch[i]
                del nmatch[i]
                rejects += 1

    print 'Rejected', rejects, 'bad matches.'
    nmatches = len(primarymatchs)
    print 'Found', nmatches, 'good matches.'

	#If no remaining matches, return empty lists
    if nmatches == 0:
       return [], [], []

    #Returns pixel scale (great circle distance in catalog [ra, dec]/cartesian distance in sextractor source [x,y])
    pixscalelist = []
    if len(primarymatchs) >= 2:
        for i in range(len(primarymatchs)-1):
            for j in range(i+1,len(primarymatchs)):
                si = primarymatchs[i]
                ci = primarymatchc[i]
                sj = primarymatchs[j]
                cj = primarymatchc[j]
                try:
                    pixscalelist.append(distance(catlist[ci],catlist[cj])/imdistance(sexlist[si],sexlist[sj]))
                except:
                    pass
        pixelscale = astrometrystats.median(pixscalelist)
        pixelscalestd = astrometrystats.stdev(pixscalelist)

        if len(primarymatchs) >= 3:
           print 'Refined pixel scale measurement: %.4f"/pix (+/- %.4f)' % (pixelscale, pixelscalestd)
        else:
           print 'Refined pixel scale measurement: %.4f"/pix' % pixelscale

	#If showmatches keyword set then print which objects match
    for i in range(len(primarymatchs)):
        si = primarymatchs[i]
        ci = primarymatchc[i]
        print  '%3i' % si, 'matches', '%3i' % ci, ' (dPA =%7.3f)' % mpa[i],
        
        #Keyword set in main program
        if showmatches:
           print
           if len(smatch[i]) < 16:
              print '  ', si, '-->', smatch[i], 
              if len(smatch[i]) >= 7: print
              print '  ', ci, '-->', cmatch[i]
           else:
              print '  ', si, '-->', smatch[i][0:10], '+', len(smatch[i])-10, 'more'
              print '  ', ci, '-->', cmatch[i][0:10], '+'#, len(cmatch[i])-10, ' more'
           if i+1 >= 10 and len(primarymatchs)-10 > 0: 
              print (len(primarymatchs)-10), 'additional matches not shown.'
              break
        else:
           print ':', str(len(smatch[i])).strip(), 'rays'
      
	#Create region files for DS9 with the sextractor sources (matchlines.im.reg) and catalog (matchlines.wcs.reg)
    out = open('matchlines.im.reg','w')
    i = -1
    color='red'
    out.write('# Region file format: DS9 version 4.0\nglobal color='+color+' font="helvetica 10 normal" select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\n')
    out.write('image\n')
    for i in range(len(primarymatchs)):
        si = primarymatchs[i]
        for j in range(len(smatch[i])):
           sj = smatch[i][j] 
           out.write("line(%.3f,%.3f,%.3f,%.3f) # line=0 0\n" % (sexlist[si].x, sexlist[si].y, sexlist[sj].x, sexlist[sj].y))
    out.close()

    out = open('matchlines.wcs.reg','w')
    i = -1
    color='green'
    out.write('# Region file format: DS9 version 4.0\nglobal color='+color+' font="helvetica 10 normal" select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\n')
    out.write('fk5\n')
    for i in range(len(primarymatchs)):
        ci = primarymatchc[i]
        for j in range(len(smatch[i])):
           cj = cmatch[i][j]
           out.write("line(%.5f,%.5f,%.5f,%.5f) # line=0 0\n" % (catlist[ci].ra, catlist[ci].dec, catlist[cj].ra, catlist[cj].dec))
    out.close()

    #future project: if not enough, go to the secondary offsets   
    
    #Returns sextractor sources and catalog sources that appear to have matches along with the mode of the position angle
    return (primarymatchs, primarymatchc, mpa)