Example #1
0
 def getsdsscat(self):
     print 'Getting SDSS spec cat for ', self.prefix
     drsearch = self.dr * 60.  #search radius in arcmin for sdss query
     #zmin=self.cz-.005
     #zmax=self.cz+.005
     #from this, we will make a field sample and a cluster sample
     #query="select n.distance,g.ra,g.dec, g.u, g.g, g.r, g.i, g.z, s.z,l.ew,l.ewErr, s.plate, s.fiberID, s.tile, g.objID,  g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z, g.petroR90_u, g.petroR90_g, g.petroR90_r, g.petroR90_i, g.petroR90_z, g.isoA_r, g.isoB_r, g.isoPhi_r, g.isoPhiErr_r, g.deVRad_r, g.deVRadErr_r, g.deVPhi_r, g.deVPhiErr_r, g.deVMag_r, g.expRad_r, g.expRadErr_r, g.expAB_r, g.expABErr_r, g.expPhi_r, g.expPhiErr_r, g.expMag_r, g.expMagErr_r, g.extinction_u,g.extinction_g,g.extinction_r,g.extinction_i,g.extinction_z, g.dered_u, g.dered_g, g.dered_r, g.dered_i, g.dered_z, g.run, g.rerun, g.camcol, g.field,g.err_u,g.err_g,g.err_r,g.err_i,g.err_z, g.rowc_u, g.rowc_g, g.rowc_r,g.rowc_i,g.rowc_z,g.colc_u,g.colc_g,g.colc_r,g.colc_i,g.colc_z from galaxy g, specobj s, specline l, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objid = s.bestobjid and g.objID = n.objID and l.specobjid = s.specobjid and s.z < %5.4f and s.z > %5.4f and (g.PrimTarget & 0x00000040) > 0 and l.LineId = 6565 order by distance" % (self.cra,self.cdec,drsearch,zmax,zmin)
     #query="select n.distance,g.ra,g.dec, g.u, g.g, g.r, g.i, g.z, s.z,l.ew,l.ewErr, s.plate, s.fiberID, s.tile, g.objID,  g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z, g.petroR90_u, g.petroR90_g, g.petroR90_r, g.petroR90_i, g.petroR90_z, g.isoA_r, g.isoB_r, g.isoPhi_r, g.isoPhiErr_r, g.deVRad_r, g.deVRadErr_r, g.deVPhi_r, g.deVPhiErr_r, g.deVMag_r, g.expRad_r, g.expRadErr_r, g.expAB_r, g.expABErr_r, g.expPhi_r, g.expPhiErr_r, g.expMag_r, g.expMagErr_r, g.extinction_u,g.extinction_g,g.extinction_r,g.extinction_i,g.extinction_z, g.dered_u, g.dered_g, g.dered_r, g.dered_i, g.dered_z, g.run, g.rerun, g.camcol, g.field,g.err_u,g.err_g,g.err_r,g.err_i,g.err_z, g.rowc_u, g.rowc_g, g.rowc_r,g.rowc_i,g.rowc_z,g.colc_u,g.colc_g,g.colc_r,g.colc_i,g.colc_z from galaxy g, specobj s, specline l, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objid = s.bestobjid and g.objID = n.objID and l.specobjid = s.specobjid and s.z < %5.4f and s.z > %5.4f and l.LineId = 6565 order by distance" % (self.cra,self.cdec,drsearch,zmax,zmin)
     # removing PrimTarget selection flag
     query = "select n.distance,g.ra,g.dec, g.u, g.g, g.r, g.i, g.z, s.z,l.ew,l.ewErr, s.plate, s.fiberID, s.tile, g.objID,  g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z, g.petroR90_u, g.petroR90_g, g.petroR90_r, g.petroR90_i, g.petroR90_z, g.isoA_r, g.isoB_r, g.isoPhi_r, g.isoPhiErr_r, g.deVRad_r, g.deVRadErr_r, g.deVPhi_r, g.deVPhiErr_r, g.deVMag_r, g.expRad_r, g.expRadErr_r, g.expAB_r, g.expABErr_r, g.expPhi_r, g.expPhiErr_r, g.expMag_r, g.expMagErr_r, g.extinction_u,g.extinction_g,g.extinction_r,g.extinction_i,g.extinction_z, g.dered_u, g.dered_g, g.dered_r, g.dered_i, g.dered_z, g.run, g.rerun, g.camcol, g.field,g.err_u,g.err_g,g.err_r,g.err_i,g.err_z, g.rowc_u, g.rowc_g, g.rowc_r,g.rowc_i,g.rowc_z,g.colc_u,g.colc_g,g.colc_r,g.colc_i,g.colc_z from galaxy g, specobj s, specline l, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objid = s.bestobjid and g.objID = n.objID and l.specobjid = s.specobjid and s.z < %5.4f and s.z > %5.4f and l.LineId = 6565" % (
         self.cra, self.cdec, drsearch, zmax, zmin)
     #        query="select n.distance,g.ra,g.dec, g.u, g.g, g.r, g.i, g.z, s.z,l.ew,l.ewErr, s.plate, s.fiberID, s.tile, g.objID  from galaxy g, specobj s, specline l, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objid = s.bestobjid and g.objID = n.objID and l.specobjid = s.specobjid and s.z < %5.4f and s.z > %5.4f and (g.PrimTarget & 0x00000040) > 0 and l.LineId = 6565 order by distance" % (self.cra,self.cdec,drsearch,zmax,zmin)#g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z, g.petroR90_u, g.petroR90_g, g.petroR90_r, g.petroR90_i, g.petroR90_z, g.isoA_r, g.isoB_r, g.isoPhi_r, g.isoPhiErr_r, g.deVRad_r, g.deVRadErr_r, g.deVPhi_r, g.deVPhiErr_r, g.deVMag_r, g.expRad_r, g.expRadErr_r, g.expABErr_r, g.expPhi_r, g.expPhiErr_r, g.expMag_r, g.expMagErr_r, g.extinction_u,g.extinction_g,g.extinction_r,g.extinction_i,g.extinction_z, g.dered_u, g.dered_g, g.dered_r, g.dered_i, g.dered_z
     #print query
     try:
         lines = sqlcl.query(query).readlines()
     except IOError:
         print "IOError for cluster", self.prefix, " trying spec query again"
         lines = sqlcl.query(query).readlines()
     print self.prefix, ": got number + 1 of spec objects = ", len(lines)
     n = homedir + 'research/LocalClusters/SDSSCatalogs/' + str(
         self.prefix) + 'galaxy.dat'
     outfile = open(n, 'w')
     j = 0
     if (len(lines) > 1.):
         for line in lines[1:]:
             if j < 0:
                 print line
                 j = j + 1
             outfile.write(line)
     outfile.close()
Example #2
0
 def getsdsscat(self):
     print 'Getting SDSS spec cat for ', self.prefix
     drsearch = 3. * 60.  #search radius in arcmin for sdss query
     #zmin=self.cz-.005
     #zmax=self.cz+.005
     #from this, we will make a field sample and a cluster sample
     query = "select n.distance,g.ra,g.dec, g.u, g.g, g.r, g.i, g.z, s.z,l.ew,l.ewErr from galaxy g, specobj s, specline l, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objid = s.bestobjid and g.objID = n.objID and l.specobjid = s.specobjid and s.z < %5.4f and s.z > %5.4f and (g.PrimTarget & 0x00000040) > 0 and l.LineId = 6565 order by distance" % (
         self.cra, self.cdec, drsearch, zmax, zmin)
     try:
         lines = sqlcl.query(query).readlines()
     except IOError:
         print "IOError for cluster", self.prefix, i, " trying spec query again"
         lines = sqlcl.query(query).readlines()
     print self.prefix, ": got number + 1 of spec objects = ", len(lines)
     n = '/home/rfinn/research/LocalClusters/SDSSCatalogs/' + str(
         self.prefix) + 'galaxy.dat'
     outfile = open(n, 'w')
     j = 0
     if (len(lines) > 1.):
         for line in lines[1:]:
             if j < 0:
                 print line
                 j = j + 1
             outfile.write(line)
     outfile.close()
Example #3
0
    def getsdss(self):
        #dA=DA(self.z[i],h100)
        #r200arcmin=self.r200[i]*1000./dA/60.
        drsearch = self.Brmax + 1.  #2xR200 in arcmin for sdss query
        query = "select g.ra, g.dec, g.isoA_r, g.isoB_r, n.distance from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and  (g.PrimTarget & 0x00000040) > 0 order by distance" % (
            self.ra, self.dec, drsearch
        )  #added flags to get rid of saturated objects, stars, etc
        try:
            lines = sqlcl.query(query).readlines()
        except IOError:
            print "IOError for cluster", self.id[
                i], i, " trying phot query again"
        lines = sqlcl.query(query).readlines()
        print "got number+1 phot objects = ", len(lines)
        out1 = open('coma-sdss-phot.dat', 'w')
        for line in lines:
            out1.write(line)
        out1.close()

        query = "select g.ra,g.dec, g.isoA_r, g.isoB_r, n.distance,  s.z from galaxy g, specobj s, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objid = s.bestobjid and g.objID = n.objID and (g.PrimTarget & 0x00000040) > 0 order by distance" % (
            self.ra, self.dec, drsearch)

        try:
            lines = sqlcl.query(query).readlines()
        except IOError:
            print "IOError for cluster", self.id[
                i], i, " trying spec query again"
        lines = sqlcl.query(query).readlines()
        out2 = open('coma-sdss-spec.dat', 'w')
        for line in lines:
            out2.write(line)
        out2.close()
Example #4
0
def getsdsscatalogs():
    drsearch = 5. * 60.  #search radius in arcmin for sdss query
    zmin = .0
    zmax = 6.
    ra = 180.
    dec = 0.
    #print i,cid[i]," ra, dec, dr, mr = %12.8f %12.8f %8.3f %5.2f" % (cra[i],cdec[i],drsearch)
    query = "select n.distance,g.ra,g.dec, g.u, g.g, g.r, g.i, g.z, s.z from galaxy g, specobj s, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objid = s.bestobjid and g.objID = n.objID and s.z < %5.4f and s.z > %5.4f and (g.PrimTarget & 0x00000040) > 0 order by distance" % (
        ra, dec, drsearch, zmax, zmin)

    print query
    try:
        lines = sqlcl.query(query).readlines()
    except IOError:
        print "IOError for cluster trying spec query again"
        lines = sqlcl.query(query).readlines()
    print "got number + 1 of spec objects = ", len(lines)
    n = 'JonGalaxy.dat'
    outfile = open(n, 'w')
    j = 0
    if (len(lines) > 1.):
        for line in lines[1:]:
            if j < 0:
                print line
                j = j + 1
            outfile.write(line)
    outfile.close()
Example #5
0
    def getsdssphotcat(self):

	print "getting phot cat for cluster",self.prefix
	drsearch=3.*60.#search radius in arcmin for sdss query
	#Vg=0.3556-0.7614*((self.avegr)-0.6148)#(V-g) from Blanton et al 2003
	#query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.plate_ID, g.MJD,  from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and (g.g < %5.2f) and ((0.384*g.g + 0.716*g.r)< %5.2f)" % (self.ra[i],self.dec[i],drsearch,(mr+1.5),mr)
	query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID,g.extinction_u, g.extinction_g, g.extinction_r, g.extinction_i, g.extinction_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and  (g.PrimTarget & 0x00000040) > 0 " % (self.cra,self.cdec,drsearch)
        try:
            lines=sqlcl.query(query).readlines()
        except IOError:
            print "IOError for cluster",self.prefix," trying phot query again"
            lines=sqlcl.query(query).readlines()

#	lines=sqlcl.query(query).readlines()
	#print query
	print "got number+1 phot objects = ",len(lines)
	#print lines

        n='/home/rfinn/research/LocalClusters/SDSSCatalogs/'+str(self.prefix)+'galaxy.photcat.dat'
        outfile=open(n,'w')
        j=0
        if (len(lines) > 1.):
            for line in lines[1:]:
                if j < 0:
                    print line
                    j=j+1
                outfile.write(line)
        outfile.close()
Example #6
0
def query_galaxies(ra,dec):
    gal_sdss_data = []
    gal_sdss_PID = []
    gal_sdss_SID = []
    for i in range(ra.size):
        ra1 = ra[i].split(':')
        dec1 = dec[i].split(':')
        coord = SkyCoord(ra1[0]+'h'+ra1[1]+'m'+ra1[2]+'s '+dec1[0]+'d'+dec1[1]+'m'+dec1[2]+'s',frame='icrs')
        print coord.ra.deg
        print coord.dec.deg
        #query = sqlcl.query("SELECT  gn.objid, ISNULL(s.specobjid,0) AS specobjid, p.ra, p.dec,p.Petromag_u-p.extinction_u AS U_mag,p.Petromag_g-p.extinction_g AS G_mag,p.Petromag_r-p.extinction_r AS R_mag,p.Petromag_i-p.extinction_i AS I_mag,p.Petromag_z-p.extinction_z AS Z_mag, ISNULL(s.z, 0) AS z, ISNULL(pz.z, 0) AS pz FROM  (Galaxy AS p JOIN dbo.fGetNearbyObjEq("+str(coord.ra.deg)+","+str(coord.dec.deg)+","+str(0.05)+") AS GN  ON p.objID = GN.objID LEFT OUTER JOIN SpecObj s ON s.bestObjID = p.objID) LEFT OUTER JOIN Photoz pz on pz.objid = p.objid WHERE p.Petromag_r-p.extinction_r < 19.1 and p.clean = 1").readlines()
        query = sqlcl.query("SELECT  gn.objid, ISNULL(s.specobjid,0) AS specobjid, p.ra, p.dec,p.Petromag_u-p.extinction_u AS U_mag,p.Petromag_g-p.extinction_g AS G_mag,p.Petromag_r-p.extinction_r AS R_mag,p.Petromag_i-p.extinction_i AS I_mag,p.Petromag_z-p.extinction_z AS Z_mag, ISNULL(s.z, 0) AS z, ISNULL(s.zErr, 0) AS z_err, ISNULL(pz.z, 0) AS pz FROM  (Galaxy AS p JOIN dbo.fGetNearbyObjEq("+str(coord.ra.deg)+","+str(coord.dec.deg)+","+str(0.05)+") AS GN  ON p.objID = GN.objID LEFT OUTER JOIN SpecObj s ON s.bestObjID = p.objID) LEFT OUTER JOIN Photoz pz on pz.objid = p.objid WHERE p.Petromag_r-p.extinction_r < 19.1").readlines()
        if len(query) > 4:
            print 'oops! More than 1 candidate found'
        if len(query) == 2:
            print 'No targets found'
            gal_sdss_data.append([0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])
            gal_sdss_PID.append(0)
            gal_sdss_SID.append(0)
            continue
        gal_sdss_data.append(map(float,query[2].split(',')))
        gal_sdss_PID.append(query[2].split(',')[0])
        gal_sdss_SID.append(query[2].split(',')[1])
        print 'Done with galaxy',i

    gal_sdss_data = np.array(gal_sdss_data)
    S_df = pd.DataFrame(gal_sdss_data,columns=['#objID','SpecObjID','ra','dec','umag','gmag','rmag','imag','zmag','spec_z','spec_z_err','photo_z'])
    S_df['#objID'] = gal_sdss_PID
    S_df['SpecObjID'] = gal_sdss_SID
    return S_df
Example #7
0
def sdss_cat(run,rerun,camcol,field):   

    field = int(field) 
    field_OR = '(' + reduce(lambda x,y: x + ' or ' + y, ['s.field=' + str(x) for x in [field,field+1,field+2,field+3,field+4,field+5]]) + ')'

    query = 'select ccFlag, t.j as psfmag_J, sqrt(1/t.jivar) as psfmagerr_J, s.psfmag_u, s.psfmag_g, s.psfmag_r, s.psfmag_i, s.psfmag_z, s.psfmagerr_u, s.psfmagerr_g, s.psfmagerr_r, s.psfmagerr_i, s.psfmagerr_z from twomass as t, star as s where t.objID=s.objID and ' + field_OR + ' and s.run=' + str(run) + ' and s.rerun=' + str(rerun) + ' and s.camcol=' + str(camcol) + ' and t.jivar != 0 and flags & dbo.fPhotoFlags(\'BLENDED\') = 0 and ccFlag = "000" '

    query = 'select  s.psfmag_u, s.psfmag_g, s.psfmag_r, s.psfmag_i, s.psfmag_z, s.psfmagerr_u, s.psfmagerr_g, s.psfmagerr_r, s.psfmagerr_i, s.psfmagerr_z from  star as s where ' + field_OR + ' and s.run=' + str(run) + ' and s.rerun=' + str(rerun) + ' and s.camcol=' + str(camcol) + ' and flags & dbo.fPhotoFlags(\'BLENDED\') = 0 '

    print query

    import sqlcl                                                    
    lines = sqlcl.query(query).readlines()
    print lines
    print lines[0]        
    keys = lines[0][:-1].split(',')
    sdss = []
    if lines[0] != 'No objects have been found':
        for l in lines[1:]:           
            d = dict(zip(keys,[float(x) for x in l.split(',')]))
            #d['stripe'] = d['dbo.fStripeOfRun(run)']
            sdss.append(d)
    else:
        sdss = [] 

    return sdss
Example #8
0
    def getsdssspeccats(self):  #get photometric sources within 2R200
        print "elapsed time = ", time.clock() - starttime
        self.mcut = N.zeros(len(self.z), 'f')
        for i in range(len(self.z)):
            dL = self.dL[i]
            print "getting spec cat for cluster abell", self.id[i]
            r200arcmin = self.r200deg[i] * 60.
            drsearch = 3. * r200arcmin  #2xR200 in arcmin for sdss query
            #Vg=0.3556-0.7614*((self.avegr)-0.6148)#(V-g) from Blanton et al 2003
            mr = mabscut - 0.1331 + 5. * N.log10(dL) + 25. + self.kcorr[i]
            print i, self.z[i], dL, mr
            self.mcut[i] = mr
            dz = 3 * self.sigma[i] / (3.e5) * (1 + self.z[i])
            zmax = self.z[i] + .5 * dz
            zmin = self.z[i] - .5 * dz
            print "ra, dec, dr, mr = %12.8f %12.8f %8.3f %5.2f" % (
                self.ra[i], self.dec[i], drsearch, mr)
            query = "select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID,g.extinction_u, g.extinction_g, g.extinction_r, g.extinction_i, g.extinction_z, l.ew, l.ewErr, l2.ew, l2.ewErr from galaxy g, specobj s, SpecLine l, SpecLine l2, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and g.objID = s.bestobjid and s.specobjID=l.specobjID and s.specobjID=l2.specobjID and (g.g < %5.2f) and  (g.PrimTarget & 0x00000040) > 0 and (s.z > %6.4f) and (s.z < %6.4f) and l.LineID = 3727 and l2.LineID = 6565" % (
                self.ra[i], self.dec[i], drsearch, (mr + 3), zmin, zmax)
            #	    query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID,g.extinction_u, g.extinction_g, g.extinction_r, g.extinction_i, g.extinction_z, l.ew, l.ewErr, l2.ew, l2.ewErr from galaxy g, specobj s, SpecLine l, SpecLine l2, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and g.objID = s.bestobjid and s.specobjID=l.specobjID and s.specobjID=l2.specobjID and (g.g < %5.2f) and  (g.PrimTarget & 0x00000040) > 0 and l.LineID = 3727 and l2.LineID = 6565" % (self.ra[i],self.dec[i],drsearch,(mr))

            lines = sqlcl.query(query).readlines()
            #print query
            print "got number+1 spec objects w/in 2R200= ", len(lines)
            #print lines
            output = "abell" + str(self.id[i]) + ".spec2r200.dat"
            outfile = open(output, 'w')
            outfile.write("#%s " % (lines[0]))
            for line in lines[1:]:
                outfile.write("%s " % (line))
            outfile.close()
def isInSDSS_DR12(ra, dec):

    querry = "select dbo.fInFootprintEq(" + str(ra) + "," + str(dec) + ", 1)"
    lines = sqlcl.query(querry).readlines()
    if lines[2] == "True\n":
        return 1
    else:
        return 0
Example #10
0
def really_isInSDSS_DR12(ra, dec):
    querry = "SELECT TOP 10 p.fieldID FROM Field AS p WHERE " + str(
        dec) + " BETWEEN p.decMin AND p.decMAx AND " + str(
            ra) + "  BETWEEN p.raMin AND p.raMax"
    lines = sqlcl.query(querry).readlines()
    if len(lines) == 2:
        return 0
    else:
        return 1
Example #11
0
def executeQ(query,res,formats): # returns a tuple of (results, modified) 
  lines = sqlcl.query(query).readlines()
  if lines[0][:-1] != '#Table1': # check if the query came back correctly
    print 'INCORRECT FORMAT RETURNED, returning previous results list'
    return (res, False)
  else:
    for l in lines[2:]:
      s = l[:-1].split(',')
      # NOTE: the formats array must exactly match the format of the returned data
      val = map(lambda x,y: x(y), formats, s)
      res.append(val)
  return (res, True)
Example #12
0
def executeQ(query,res): # returns a tuple of (results, modified) 
  lines = sqlcl.query(query).readlines()
  if lines[0][:-1] != '#Table1': # check if the query came back correctly
    print 'INCORRECT FORMAT RETURNED, returning previous results list'
    return (res, False)
  else:
    for l in lines[2:]:
      s = l[:-1].split(',')
      # NOTE: this line is heavily dependant on what we query, and in what order
      val = [float(s[0]),float(s[1]),int(s[2]),int(s[3]),float(s[4]),int(s[5]),float(s[6]),float(s[7]),float(s[8]),float(s[9])]
      res.append(val)
  return (res, True)
Example #13
0
    def dataCollect(self):
        import os

        queries = [self.sql]
        url = os.getenv("SQLCLURL", SqlReader.default_url)
        fmt = SqlReader.default_fmt


        # Run all queries sequentially
        for qry in queries:
            file = sqlcl.query(qry, url, fmt)
            self.fileWrite(file)
Example #14
0
 def getsdssspeccat(self):
     print 'Getting SDSS spec cat for ',self.prefix
     drsearch=3.*60.#search radius in arcmin for sdss query
     #zmin=self.cz-.005
     #zmax=self.cz+.005
     #from this, we will make a field sample and a cluster sample
     query="select n.distance,g.ra,g.dec, g.u, g.g, g.r, g.i, g.z, s.z,l.ew,l.ewErr, s.plate, s.fiberID, s.tile from galaxy g, specobj s, specline l, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objid = s.bestobjid and g.objID = n.objID and l.specobjid = s.specobjid and s.z < %5.4f and s.z > %5.4f and (g.PrimTarget & 0x00000040) > 0 and l.LineId = 6565 order by distance" % (self.cra,self.cdec,drsearch,zmax,zmin)
     try:
         lines=sqlcl.query(query).readlines()
     except IOError:
         print "IOError for cluster",self.prefix," trying spec query again"
         lines=sqlcl.query(query).readlines()
     print self.prefix,": got number + 1 of spec objects = ",len(lines)
     n='/home/rfinn/research/LocalClusters/SDSSCatalogs/'+str(self.prefix)+'galaxy.dat'
     outfile=open(n,'w')
     j=0
     if (len(lines) > 1.):
         for line in lines[1:]:
             if j < 0:
                 print line
                 j=j+1
             outfile.write(line)
     outfile.close()
Example #15
0
    def getsdssphotcats(self):  #get photometric sources within 2R200 
        print "elapsed time = ",time.clock()-starttime
        self.mcut=N.zeros(len(self.z),'f')
        cl=N.arange(17,len(self.z),1)
	self.nphot=N.zeros(len(self.z),'f')
	self.nspec=N.zeros(len(self.z),'f')
        for i in range(len(self.z)):
        #for i in cl:
            dL = self.dL[i]
            print "getting phot cat for cluster abell",self.id[i]
            r200arcmin=self.r200deg[i]*60.
            #drsearch=2.*r200arcmin#2xR200 in arcmin for sdss query
            drsearch=1.*r200arcmin#2xR200 in arcmin for sdss query
            #Vg=0.3556-0.7614*((self.avegr)-0.6148)#(V-g) from Blanton et al 2003
            mr=mabscut - 0.1331 + 5.*N.log10(dL)+25.+self.kcorr[i]
            print i, self.z[i], dL, mr
            self.mcut[i]=mr
            print "ra, dec, dr, mr = %12.8f %12.8f %8.3f %5.2f" % (self.ra[i],self.dec[i],drsearch,mr)
            #query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.plate_ID, g.MJD,  from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and (g.g < %5.2f) and ((0.384*g.g + 0.716*g.r)< %5.2f)" % (self.ra[i],self.dec[i],drsearch,(mr+1.5),mr)
            query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID,g.extinction_u, g.extinction_g, g.extinction_r, g.extinction_i, g.extinction_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and (g.g < %5.2f) and  (g.PrimTarget & 0x00000040) > 0 " % (self.ra[i],self.dec[i],drsearch,(mr))
            lines=sqlcl.query(query).readlines()
            #print query
            print "got number+1 phot objects = ",len(lines)
            #print lines
	    self.nphot[i]=1.*len(lines)

	    query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID,g.extinction_u, g.extinction_g, g.extinction_r, g.extinction_i, g.extinction_z, l.ew, l.ewErr, l2.ew, l2.ewErr from galaxy g, specobj s, SpecLine l, SpecLine l2, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and g.objID = s.bestobjid and s.specobjID=l.specobjID and s.specobjID=l2.specobjID and (g.g < %5.2f) and  (g.PrimTarget & 0x00000040) > 0 and l.LineID = 3727 and l2.LineID = 6565" % (self.ra[i],self.dec[i],drsearch,(mr))


            lines=sqlcl.query(query).readlines()
            #print query
            print "got number+1 spec objects w/in R200= ",len(lines)
            #print lines
	    self.nspec[i]=1.*len(lines)

	self.compl=self.nspec/self.nphot
	print "average completeness of sdss spectroscopy is = ",N.average(self.compl), pylab.std(self.compl)
Example #16
0
def getSDSSfields(ra, dec, size):  # all in degree
  
  
  delta = 0.6*size+0.2
  
  #if size > 1:
      #delta = 1.2*size+0.4
  
  ra_max  =  ra+2*delta
  ra_min  =  ra-2*delta
  dec_max =  dec+delta
  dec_min =  dec-delta
  
  querry = """
  
 SELECT
 fieldID,
 run, 
 camCol, 
 field,
 ra, 
 dec,
 run,
 rerun 
 FROM Field   
  """
  
  querry += "WHERE ra BETWEEN "+str(ra_min)+" and "+str(ra_max)+" and dec BETWEEN "+str(dec_min)+" and "+str(dec_max)
  
  print querry
  
  lines = sqlcl.query(querry).readlines()
  N = len(lines)

  
  field_lst = []
  for i in np.arange(2,N):
      line = lines[i]
      line = line.split(',')
      run    = line[1]
      camcol = line[2]
      field  = line[3]
      ra_    = line[4]
      dec_   = line[5]
      field_lst.append([run, camcol, field])
  
  
  return field_lst
Example #17
0
 def dataQuery(self, myquery):
     """ Return a table or print a error of data query from SkyServer of SDSS.
     """
     print 'Quering the SkySever'
     page =  sqlcl.query(myquery, fmt='html').read()
     soup = BeautifulSoup(page)
     tables = [table for table in soup.findAll("table")]    
     if (len(tables) >=1):
         print 'Data recived'
         return tables
     else:
         h3 = [h for h in soup.findAll("h3")]
         error =  str(h3[1])
         error = replace(error,'<h3 bgcolor=\"pink\"><font color=\"red\">', '' )
         error = replace(error,'</font></h3>', '' )
         error = replace(error, '<br />', '\n')
         print error
         return (None,error)
def remote_search_field(ra, dec, radius, use_all=False, extra=0.05):
    """
    Given the Ra, DEC for the center of the field, the radius of the search area
    in unit of degree, returns the information for the FIELDS that cover this
    search area.

    The remote search use SQLCL to access the SDSS on-line database;
    The search is not very accurate

    @use_all : If True, all fields, including the ones with primaryArea==0, will
    be returned.
    """
    if (ra < 0.0) or (ra > 360.0):
        raise Exception("RA should be between 0 and 360 degree!")
    if (dec < -90.0) or (dec > 90.0):
        raise Exception("Dec should be between -90 and 90 degree!")
    if radius > 1.0:
        warning = "Radius is too large!! Be careful!! (Radius < 1.0deg)"
        highlight_output(warning)

    query = define_query(ra, dec, radius, use_all = use_all, extra=extra)
    result = sqlcl.query(query).readlines()

    n_field = (len(result) - 2)
    if n_field <= 0:
        raise Exception("No useful field is returned!! Check!!")
    else:
        result = result[2:]

    data = []
    for ii in result:
        line = ii.replace("\n"," ")
        temp = np.genfromtxt(StringIO(line), delimiter=",", dtype=None)
        data.append(temp)

    dtype=[('fieldID', int), ('run', int), ('rerun', int), ('camcol', int),
           ('field', int), ('quality', int), ('score', float), ('ra', float),
           ('dec', float), ('raMin', float), ('raMax', float),
           ('decMin', float), ('decMax', float), ('nGalaxy', int),
           ('nStars', float), ('primaryArea', float)]

    result = np.array(data, dtype=dtype)

    return result
def remote_search_field(ra,dec,radius,use_all=False,extra=None):
    """
    Given the Ra, DEC for the center of the field, the radius of the search area
    in unit of degree, returns the information for the FIELDS that cover this
    search area.

    The remote search use SQLCL to access the SDSS on-line database;
    The search is not very accurate

    @use_all : If True, all fields, including the ones with primaryArea==0, will
    be returned.
    """
    if (ra<0.0) or (ra>360.0):
        raise Exception("RA should be between 0 and 360 degree!")
    if (dec<-90.0) or (dec>90.0):
        raise Exception("Dec should be between -90 and 90 degree!")
    if radius > 1.0:
        warning = "Radius is too large!! Be careful!! (Radius<1.0deg)"
        highlight_output(warning)

    query = define_query(ra,dec,radius,use_all=use_all,extra=extra)
    result = sqlcl.query(query).readlines()

    n_field = (len(result)-2)
    if n_field == 0:
        raise Exception("No useful field is returned!! Check!!")

    data = []
    for ii in result[2:]:
        line = ii.replace("\n"," ")
        data.append(line.split(','))
    result = np.recarray(data, dtype=[('fieldid', int), ('run', int),
                                      ('rerun', int), ('camcol', int),
                                      ('field', int), ('quality', str),
                                      ('score', float), ('ra', float),
                                      ('dec', float), ('primaryarea', float)])
    print result.shape

    # TODO: Not working perfectly right now; find a way to work around ,
    #       Maybe simply use a different format

    return result
 def query(self,query):
     result = sqlcl.query(query).readlines()
     data =[]
     count =0
     if (DEBUG): print (result)
     for i in result:
         if count>1:
             list =i.split(',')
             if (len(list)>2):
                 list[2]= list[2][:-1]
                 data.append(list)
         count += 1 
     if (DEBUG):print (result)
     if (len(data)>0):
         if (len(data[0])>0):
             while (data[0][0][1:6]=="ERROR"):
                 #Case where doing more than 60 queries in 1 minute
                 print("ERROR: Too much query in 1 minute. Sleep for 60 second.")
                 time.sleep(60)
                 data = self.query(query)
     return (data)
Example #21
0
def executeQueries(queries, formats):
	#run those queries yo
	first = True
	results = []
	count = 1

	for q in queries:
		print 'running query ' + `count`
		count = count + 1
		lines = sqlcl.query(q).readlines()
		if lines[0][:-1] != '#Table1':
			# the query failed, just print it and exit
			print 'the query failed, moving to next query.'
		else:
			# if first: #add fields
			# 	results.append(lines[1][:-1].split(','))
			# 	first = False
			for l in lines[2:]:
				results.append(map(lambda x,y: x(y), formats ,l[:-1].split(',')))

	return results
Example #22
0
    def getSDSSfields(ra, dec, size):  # all in degree
        delta = size
        ra_max = ra + (delta / np.cos(abs(np.radians(dec))))
        ra_min = ra - (delta / np.cos(abs(np.radians(dec))))
        dec_max = dec + delta
        dec_min = dec - delta

        querry = """

         SELECT
         fieldID,
         run, 
         camCol, 
         field,
         ra, 
         dec,
         run,
         rerun 
         FROM Field   
         """

        querry += "WHERE ra BETWEEN " + str(ra_min) + " and " + str(
            ra_max) + " and dec BETWEEN " + str(dec_min) + " and " + str(
                dec_max)

        lines = sqlcl.query(querry).readlines()
        N = len(lines)

        field_lst = []
        for i in np.arange(2, N):
            line = str(lines[i])
            line = line.split(',')
            run = line[1]
            camcol = line[2]
            field = line[3]
            ra_ = line[4]
            dec_ = line[5]
            field_lst.append([run, camcol, field])

        return field_lst
def get_sdss_photometry(coords):
    # print len(coords)
    if len(coords) == 1 or len(coords) != 4:
        print "It checks the SDSS PhotoObjAll catalog to find all photometric objects\n within a given radius."
        print "usage : sdss_photo_check.py (ra) (dec) -r (search radius)"
        print "\tra : degree"
        print "\tdec : degree"
        print "\tband : filter (ugriz)"
        print "\tsearch radius : arcsec (optional) (default : 3 arcsec)"
        print "output : ObjId model_u model_g model_r model_i model_z"
        sys.exit()

        # print coords, "****************************", coords[2]
    ra = str(coords[0])
    dec = str(coords[1])
    band = coords[2]
    search_rad = str(coords[3])

    sql_query = (
        "select P.modelMag_"
        + band
        + " from PhotoObjAll P, dbo.fGetNearbyObjAllEq("
        + ra
        + ","
        + dec
        + ","
        + search_rad
        + ") n where P.objID = n.objID"
    )
    query_result = sqlcl.query(sql_query).readlines()
    if len(query_result) > 1:
        data_part = string.split(query_result[1], ",")
        output_string = ""
        for x in data_part:
            output_string = output_string + x.strip() + " "
        print output_string
        time.sleep(1.0)
    else:
        print "No object found"
def photoobj_search(ra, dec, radius, band='r', use_all=True, extra=0.05):
    """
    Searh for photometric information of all objects within certain distance
    between the central RA, DEC

    """

    if (ra < 0.0) or (ra > 360.0):
        raise Exception("RA should be between 0 and 360 degree!")
    if (dec < -90.0) or (dec > 90.0):
        raise Exception("Dec should be between -90 and 90 degree!")
    if radius > 1.0:
        warning = "Radius is too large!! Be careful!! (Radius < 1.0deg)"
        highlight_output(warning)

    query = photo_query(ra, dec, radius, band=band, use_all = use_all,
                        extra=extra)
    result = sqlcl.query(query).readlines()

    n_field = (len(result) - 2)
    if n_field <= 0:
        raise Exception("No useful field is returned!! Check!!")
    else:
        result = result[2:]

    data = []
    for ii in result:
        line = ii.replace("\n", "")
        temp = np.genfromtxt(StringIO(line), delimiter=",", dtype=None)
        data.append(temp)

    dtype = [('objID', int), ('ra', float), ('dec', float), ('type', int),
             ('clean', int), ('nChild', int), ('petroR90', float),
             ('psfMag', float), ('cModelMag', float), ('expAB', float),
             ('expPhi', float), ('devAB', float), ('devPhi', float)]
    table = np.array(data, dtype)

    return table
Example #25
0
    def getsdssphotcats(self):  #get photometric sources within 2R200
        print "elapsed time = ", time.clock() - starttime
        self.mcut = N.zeros(len(self.z), 'f')
        cl = N.arange(17, len(self.z), 1)
        for i in range(len(self.z)):
            #for i in cl:
            dL = self.dL[i]
            print "getting phot cat for cluster abell", self.id[i]
            r200arcmin = self.r200deg[i] * 60.
            #drsearch=2.*r200arcmin#2xR200 in arcmin for sdss query
            drsearch = 3. * r200arcmin  #2xR200 in arcmin for sdss query
            #Vg=0.3556-0.7614*((self.avegr)-0.6148)#(V-g) from Blanton et al 2003
            mr = mabscut - 0.1331 + 5. * N.log10(dL) + 25. + self.kcorr[i]
            print i, self.z[i], dL, mr
            self.mcut[i] = mr
            print "ra, dec, dr, mr = %12.8f %12.8f %8.3f %5.2f" % (
                self.ra[i], self.dec[i], drsearch, mr)
            #query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.plate_ID, g.MJD,  from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and (g.g < %5.2f) and ((0.384*g.g + 0.716*g.r)< %5.2f)" % (self.ra[i],self.dec[i],drsearch,(mr+1.5),mr)
            query = "select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID,g.extinction_u, g.extinction_g, g.extinction_r, g.extinction_i, g.extinction_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and (g.g < %5.2f) and  (g.PrimTarget & 0x00000040) > 0 " % (
                self.ra[i], self.dec[i], drsearch, (mr))
            #line from sdssinter.py code
            #query="select n.distance from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and g.r< %5.2f and  (g.PrimTarget & 0x00000040) > 0 order by distance" % (self.ra[i],self.dec[i],drsearch,mr)#added flags to get rid of saturated objects, stars, etc

            #query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID" % (self.ra[i],self.dec[i],drsearch)#no mag cut

            #query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and g.r<17.7 and g.g<18.0" % (self.ra[i],self.dec[i],drsearch)
            lines = sqlcl.query(query).readlines()
            #print query
            print "got number+1 phot objects = ", len(lines)
            #print lines
            output = "abell" + str(self.id[i]) + ".phot.dat"
            outfile = open(output, 'w')
            outfile.write("#%s " % (lines[0]))
            for line in lines[1:]:
                outfile.write("%s " % (line))
            outfile.close()
Example #26
0
lowlim = float(sys.argv[3])
highlim = float(sys.argv[4])

fullcat_name_path = os.path.join(CURRENT_DIR, "Full_SDSS.dat")
trimmedcat_name_path = os.path.join(CURRENT_DIR, "Trimmed_SDSS.dat")
fullcat = open(fullcat_name_path, 'w')
trimmedcat = open(trimmedcat_name_path, 'w')

query = """
SELECT TOP """ + str(Num) + """
cast(str(p.ra,13,8) as float) as ra,cast(str(p.[dec],13,8) as float) as dec,p.psfMag_u,p.psfMag_g,p.psfMag_r,p.psfMag_i,p.psfMag_z,p.psfMagErr_u,p.psfMagErr_g,p.psfMagErr_r,p.psfMagErr_i,p.psfMagErr_z,dbo.fIAUFromEq(p.ra,p.[dec]) as SDSSname 
FROM ..PhotoObj AS p
""" + "JOIN dbo.fGetNearbyObjEq(" + str(RA) + "," + str(DEC) + "," + str(
    Area) + ") AS b ON b.objID = P.objID"

data = sqlcl.query(query).read()
#print data
print >> fullcat, str(data)
fullcat.close()
############################### slim down #########################

table = numpy.genfromtxt(fullcat_name_path,
                         delimiter=',',
                         dtype=str,
                         skip_header=2,
                         unpack=True)
RA_star, DEC_star, psfMag_u, psfMag_g, psfMag_r, psfMag_i, psfMag_z, psfMagErr_u, psfMagErr_g, psfMagErr_r, psfMagErr_i, psfMagErr_z, SDSSname = table[:]

RA_star = numpy.array(RA_star, dtype=float)
DEC_star = numpy.array(DEC_star, dtype=float)
psfMag_u = numpy.array(psfMag_u, dtype=float)
Example #27
0
import sqlcl
import sys, os
from csvMathMod import csv2math

# Path for transfer files
prefix = "/ram/"
# prefix for transfer files
xfer = "xfer-"

if len(sys.argv) != 2 :
   print('Syntax is : python sdss_query.py "SQL query"')
   exit(0)
print("Executing Query : "+sys.argv[1]+"\n")

lines = sqlcl.query(sys.argv[1]).readlines()
# New versions seem to produce an extra first line with Table so remove
print lines[1:]
os.system("rm "+prefix+xfer+"* 2> /dev/null")
csv2math(lines[1:])
#!/usr/bin/env python
import numpy as np
import sqlcl
from StringIO import StringIO
ra = 239.583329
dec = 0  #27.233413
rad = 20.0
radians = (rad / 60.0) * np.pi / 180.0
solid_angle = 2.0 * np.pi * (1.0 - np.cos(radians))
solid_angle2 = 2.0 * np.pi * (radians**2 / 2.0)
area = np.pi * rad**2
print "radians: %f solid angle: %e solid angle2: %e square deg: %f " % (
    radians, solid_angle, solid_angle2, solid_angle2 / (np.pi * 4) * 41253)
print "sq arcmin: %f sq degree: %f" % (np.pi * rad**2, np.pi * (rad / 60.0)**2)
result = sqlcl.query(
    "select p.ra, p.dec from PhotoObjAll p,  dbo.fGetNearbyObjEq(%f,%f,%f) as r where p.ObjID = r.ObjID"
    % (ra, dec, rad)).read()

datagal = np.genfromtxt(StringIO(result), names=True, delimiter=",")
print datagal['dec'].size
print "number of elements per sq arcmin: %f" % (datagal['dec'].size / area)
ra1 = datagal['ra'].min()
ra2 = datagal['ra'].max()
dec1 = datagal['dec'].min()
dec2 = datagal['dec'].max()

print ra
print "min ra: %f max ra: %f diff: %f diff arcmin: %f" % (ra1, ra2,
                                                          (ra2 - ra1),
                                                          (ra2 - ra1) * 60.0)
print "min dec: %f max dec: %f diff: %f diff arcmin: %f" % (dec1, dec2,
Example #29
0
    def getsdssphotcat(self):
        print 'Getting SDSS phot cat for ', self.prefix
        drsearch = self.dr * 60.  #search radius in arcmin for sdss query
        #zmin=self.cz-.005
        #zmax=self.cz+.005
        #from this, we will make a field sample and a cluster sample

        flag = 0
        nrun = 1
        print 'getting to while loop'
        while flag == 0:
            print 'inside while loop'
            #Vg=0.3556-0.7614*((self.avegr)-0.6148)#(V-g) from Blanton et al 2003
            #query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.plate_ID, g.MJD,  from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and (g.g < %5.2f) and ((0.384*g.g + 0.716*g.r)< %5.2f)" % (self.ra[i],self.dec[i],drsearch,(mr+1.5),mr)
            #changed so that only galaxies w/out spectra are returned
            #query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID, g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z, g.petroR90_u, g.petroR90_g, g.petroR90_r, g.petroR90_i, g.petroR90_z, g.isoA_r, g.isoB_r, g.isoPhi_r, g.isoPhiErr_r, g.deVRad_r, g.deVRadErr_r, g.deVPhi_r, g.deVPhiErr_r, g.deVMag_r, g.expRad_r, g.expRadErr_r, g.expAB_r, g.expABErr_r, g.expPhi_r, g.expPhiErr_r, g.expMag_r, g.expMagErr_r, g.extinction_u,g.extinction_g,g.extinction_r,g.extinction_i,g.extinction_z, g.dered_u, g.dered_g, g.dered_r, g.dered_i, g.dered_z,  g.run, g.rerun, g.camcol, g.field, g.err_u,g.err_g,g.err_r,g.err_i,g.err_z,g.rowc_u, g.rowc_g, g.rowc_r,g.rowc_i,g.rowc_z,g.colc_u,g.colc_g,g.colc_r,g.colc_i,g.colc_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and  (g.PrimTarget & 0x00000040) > 0 and (g.specObjID = 0)" % (self.cra,self.cdec,drsearch)
            query = "select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID, g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z, g.petroR90_u, g.petroR90_g, g.petroR90_r, g.petroR90_i, g.petroR90_z, g.isoA_r, g.isoB_r, g.isoPhi_r, g.isoPhiErr_r, g.deVRad_r, g.deVRadErr_r, g.deVPhi_r, g.deVPhiErr_r, g.deVMag_r, g.expRad_r, g.expRadErr_r, g.expAB_r, g.expABErr_r, g.expPhi_r, g.expPhiErr_r, g.expMag_r, g.expMagErr_r, g.extinction_u,g.extinction_g,g.extinction_r,g.extinction_i,g.extinction_z, g.dered_u, g.dered_g, g.dered_r, g.dered_i, g.dered_z,  g.run, g.rerun, g.camcol, g.field, g.err_u,g.err_g,g.err_r,g.err_i,g.err_z,g.rowc_u, g.rowc_g, g.rowc_r,g.rowc_i,g.rowc_z,g.colc_u,g.colc_g,g.colc_r,g.colc_i,g.colc_z from galaxy g where g.r < 22 and g.ra > %12.8f and g.ra < %12.8f and g.dec > %12.8f and g.dec < %12.8f and (g.specObjID = 0)" % (
                self.cra - drsearch / 2., self.cra + drsearch / 2.,
                self.cdec - drsearch / 2., self.cdec + drsearch / 2.)
            #query="select g.ra, g.dec from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and  (g.PrimTarget & 0x00000040) > 0 and (g.specObjID = 0)" % (self.cra,self.cdec,drsearch)#changed so that only galaxies w/out spectra are returned
            # the following timed out in 10 min
            #query="select count(*) from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and  (g.specObjID = 0)" % (self.cra,self.cdec,drsearch)#changed so that only galaxies w/out spectra are returned
            # the following timed out at 10 min
            #query="select count(*) from galaxy g where (g.ra between %12.8f and %12.8f) and (g.dec between %12.8f and %12.8f) and (g.PrimTarget & 0x00000040) > 0 and (g.specObjID = 0)" % (self.cra-drsearch,self.cra+drsearch,self.cdec-drsearch,self.cdec+drsearch)
            # now trying this
            #query="select count(*) from galaxy g where (g.ra between %12.8f and %12.8f) and (g.dec between %12.8f and %12.8f) and (g.specObjID = 0)" % (self.cra-drsearch,self.cra+drsearch,self.cdec-drsearch,self.cdec+drsearch)
            # sdss website says the following query completes in 18 sec.  let's see how it does...
            #query='SELECT p.ra, p.dec, p.ModelMag_i, p.extinction_i FROM TargetInfo t, PhotoTag p WHERE (t.primtarget & 0x00000006>0) and p.objid=t.targetobjid'
            # this does, in fact, complete very quickly!

            #query="select count(*) from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and (g.PrimTarget & 0x00000040) > 0 and (g.specObjID = 0)" % (self.cra,self.cdec,drsearch)

            #        query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID, g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z, g.petroR90_u, g.petroR90_g, g.petroR90_r, g.petroR90_i, g.petroR90_z, g.isoA_r, g.isoB_r, g.isoPhi_r, g.isoPhiErr_r, g.deVRad_r, g.deVRadErr_r, g.deVPhi_r, g.deVPhiErr_r, g.deVMag_r, g.expRad_r, g.expRadErr_r, g.expAB_r, g.expABErr_r, g.expPhi_r, g.expPhiErr_r, g.expMag_r, g.expMagErr_r, g.extinction_u,g.extinction_g,g.extinction_r,g.extinction_i,g.extinction_z, g.dered_u, g.dered_g, g.dered_r, g.dered_i, g.dered_z,  g.run, g.rerun, g.camcol, g.field, g.err_u,g.err_g,g.err_r,g.err_i,g.err_z,g.rowc_u, g.rowc_g, g.rowc_r,g.rowc_i,g.rowc_z,g.colc_u,g.colc_g,g.colc_r,g.colc_i,g.colc_z from galaxy g where (g.ra between %12.8f and %12.8f) and (g.dec between %12.8f and %12.8f) and (g.PrimTarget & 0x00000040) > 0 and (g.specObjID = 0)" % (self.cra-drsearch,self.cra+drsearch,self.cdec-drsearch,self.cdec+drsearch)#changed so that only galaxies w/out spectra are returned
            print query
            start_time = time.time()
            try:
                lines = sqlcl.query(query).readlines()
            except IOError:
                print "IOError for cluster", self.prefix, " trying phot query again"
                lines = sqlcl.query(query).readlines()

            elapsed_time = time.time() - start_time
            print 'time to execute query = ', elapsed_time, ' sec, ', elapsed_time / 60., ' min'
            print "got number+1 phot objects = ", len(lines)

            n = homedir + 'research/LocalClusters/SDSSCatalogs/' + str(
                self.prefix) + 'galaxy.photcat.dat'
            outfile = open(n, 'w')
            j = 0
            flag = 1
            if (len(lines) > 1.):
                for line in lines[1:]:
                    if j < 0:
                        print line
                        j = j + 1
                    outfile.write(line)
                    if line.find('Server.ScriptTimeout') > -1:
                        flag = 0
                    elif line.find('Timeout') > -1:
                        flag = 0
            outfile.close()
            nrun += 1
            if nrun > 15:
                return
            if flag == 0:
                print self.prefix
                print 'Running query again b/c of ScriptTimeout'
                print 'starting attempt = ', nrun
    def source_info(self,r_fits_filename):
        '''
        [ra,dec,margin,radius,pgc] ==> Is margin info necessary(?) YES
        Input: Filename String of R band Mosaic fit file
        Returns the updated [ra,dec,margin,radius,pgc] info about the identified RC3 source as a list
        If no RC3 source is identified then ['@','@',margin_value,'@','@'] is returned
        If RC3 lie outside of SDSS footprint then [-1,-1,-1,-1,-1] is returned
        '''
        try:
            updated = open("rc3_updated.txt",'a') 
            self.num_iterations +=1
            print ("{}th iteration".format(self.num_iterations))
            if (self.num_iterations < 3): #5 is too much
                print ("------------------source_info----------------------")
                file = r_fits_filename 
                print("Source info for {}".format(file))
                if (file ==-1): #special value reserved for not in SDSS footprint galaxies
                    return [-1,-1,-1,-1,-1]

            # File info 
                hdulist = pyfits.open(file)
                rc3_ra= hdulist[0].header['RA']
                rc3_dec= hdulist[0].header['DEC']
                rc3_radius = hdulist[0].header['RADIUS']
                margin = hdulist[0].header['MARGIN']
                pgc = hdulist[0].header['PGC']

            	#Source Extraction
                os.system("sex {} -c default.sex".format(file))
            	# A list of other RC3 galaxies that lies in the field
                # In the case of source confusion, find all the rc3 that lies in the field.
                other_rc3s = sqlcl.query("SELECT distinct rc3.ra, rc3.dec FROM PhotoObj as po JOIN RC3 as rc3 ON rc3.objid = po.objid  WHERE po.ra between {0}-{1} and  {0}+{1} and po.dec between {2}-{3} and  {2}+{3}".format(str(rc3_ra),str(margin),str(rc3_dec),str(margin))).readlines()
                print (other_rc3s)
                data =[]
                count =0
                for i in other_rc3s:
                    if count>1:
                        list =i.split(',')
                        list[0] = float(list[0])
                        list[1]= float(list[1][:-1])
                        data.append(list)
                    count += 1 
                print ("ra,dec of catalog sources")
                rc3_data = map (np.array,data)
                print ("rc3_data: "+str(rc3_data))
                # if (len(rc3_data)>0):
                # print ("here2")
                distances=[]
                for i in range(len(data)-1):#len(data)//2):
                    if (len(data)>1 ): #odd number (unpaired) RC3s that lie in the field is ignored for now 
                    # (but we have to take it into consideration eventually)
                    # and len(data)%2==0
                        d2p= np.array(data[i])-np.array(data[i+1])
                        print ("d2p: {}".format(d2p))
                        distances.append(d2p)	    
                if(len(distances)!=0):
                    print (distances)
                if (len(distances)>1):
                    print ("More than 2 galaxies inside field!")   
                print (distances)         	                
          		#Conduct pairwise comparison
                catalog = open("test.cat",'r')
                #Creating a list of radius
                radius_list = []
                # Creating a corresponding list of ra,dec
                #sextract = []
                sextract_dict ={}
                for line in catalog:
                    # print (line)
                    line = line.split()
                    if (line[0]!='#'):
                        # print("HERE!")
                    	#sextract.append(np.array([line[2],line[3]]))
                        radius=np.sqrt((float(line[6])-float(line[4]))**2+(float(line[7])-float(line[5]))**2)/2
                        #print(radius)
                        radius_list.append(radius)
                        coord = np.array([float(line[2]),float(line[3])])
                        sextract_dict[radius]=coord
                print ("Radius: "+str(radius_list))
                print ("SExtract_dict: "+str(sextract_dict))
                if (len(sextract_dict)>0):
                    #special value that indicate empty list (no object detected by SExtractor)
                    radii='@'
                    new_ra='@'
                    new_dec='@'
                    catalog = open("test.cat",'r')
                    n=-1
                    if (len(distances)!=0):
                        # if there is source confusion, then we want to keep the nth largest radius
                        print ("Source Confusion")
                        n=len(distances)+1
                        print ("sextract_dict:")
                        print (sextract_dict)
                        print ("N-th largest radius:")
                        print(heapq.nlargest(n,sextract_dict))
                        #nth largest radius
                        nth_largest=heapq.nlargest(n,sextract_dict)
                        sextract=[]
                        for i in heapq.nlargest(n,sextract_dict):
                            sextract.append(np.array(sextract_dict[i]))
                        print ("sextract:")
                        print (sextract)

                        # radius
                        nth_largest=[i for i in nth_largest if float(i)>15.]
                        print(nth_largest)
                        if(len(nth_largest)!=0):
                            radii = nth_largest[0]

                        #Coordinate matching by pairs
                        diff = []
                        #all possible coordinate pairs 
                        coord_match=[]
                        for i in rc3_data : 
                            #determine shift vector 
                            for j in sextract:
                                print (str(i)+" " +str(j))
                                coord_match.append([i,j])
                                diff.append((j-i).tolist())  
                        print ("coord_match: "+str(coord_match))     
                        print ("diff: "+str(diff))
                        abs_diff = map (lambda x : map(lambda y:abs(y), x), diff)        
                        print ("abs_diff: "+str(abs_diff))
                        tmp = heapq.nsmallest(n,abs_diff)
                        print ("tmp : "+str(tmp))
                        # Bascially doing this , the long way, becasuse Python apparently can not do list -by element comparison and complains
                        #inx=abs_diff.index(np.array(i))
                        inx=[]
                        for i in tmp:
                            for j in abs_diff:
                                #print (i)
                                #print (j)
                                if (i==j):
                                    print (abs_diff.index(j))
                                    inx.append(abs_diff.index(j))
                        print (inx)
                        matched=[]
                        for i in inx:
                            print coord_match[i]
                            matched.append(coord_match[i])
                            # for j in coord_match:
                            #     print ([i,j])
                            #     if (all(np.array([1,2])==np.array([1,2])))
                            # #matches = [coord for coord in coord_match ]
                            # #print (matches)
                         #        print ("Matched coordinates: "+str(coord_match[inx]))

                        # A list of other RC3 galaxies that lies in the field
                        other_rc3s = sqlcl.query("SELECT distinct rc3.pgc,rc3.ra,rc3.dec FROM PhotoObj as po JOIN RC3 as rc3 ON rc3.objid = po.objid  WHERE po.ra between {0}-{1} and  {0}+{1} and po.dec between {2}-{3} and  {2}+{3}".format(str(rc3_ra),str(margin),str(rc3_dec),str(margin))).readlines()
                        print ("PGC of other_rc3s")
                        print (other_rc3s)

                        info ={}
                        count =0
                        for i in other_rc3s:
                            if count>1:
                                list =i.split(',')
                                pgc = int(list[0][6:])
                                ra= float(list[1][:-1])
                                dec= float(list[2][:-1])
                                info[pgc]= [ra,dec]
                            count += 1 
                        print (info)
                        print ("The galaxy that we want to mosaic is: "+str(info[self.pgc]))
                        new_ra= info[self.pgc][0]
                        new_dec = info[self.pgc][1]
                    else:
                        print ("Source is Obvious")
                        n=1 # if no source confusion then just keep the maximum radius
                        catalog = open("test.cat",'r')
                        #Creating a list of radius
                        radius = []
                        for line in catalog:
                            #print (line)
                            line = line.split()
                            if (line[0]!='#'):
                                radius.append(np.sqrt((float(line[6])-float(line[4]))**2+(float(line[7])-float(line[5]))**2)/2)
                        #special value that indicate empty list (no object detected by SExtractor)
                        radii='@'
                        new_ra='@'
                        new_dec='@'
                        catalog = open("test.cat",'r')
                        # If there is no other RC3 in the field, it means the largest galaxy in the field is the RC3 we are interested in
                        # So find max radius and treat as if it is rc3
                        for i in catalog:
                            line = i.split()
                            if (line[0]!='#' ):
                                radii = np.sqrt((float(line[6])-float(line[4]))**2+(float(line[7])-float(line[5]))**2)/2
                                if (radii==max(radius)):
                                    print ('Biggest Galaxy with radius {} pixels!'.format(str(radii)))
                                    radii = radii
                                    new_ra= line[2]
                                    new_dec = line[3]
                                    break
                    print ("new_ra and new_dec: {} , {}  ".format(str(new_ra),str(new_dec)))
                    if (radii!='@' and float(radii)>15): # There exist 1 or more detected source
                        print ("Radii: {} pixel".format(str(radii)))
                        radii = 0.00010995650106797878*radii #pixel to degree conversion
                        print ("Radii: {} degrees".format(str(radii)))
                        print ("rc3: {} , updated: {} ".format(rc3_ra, new_ra))
                        print ("rc3: {} , updated: {} ".format(rc3_dec,new_dec))
                        print ("rc3: {} , updated: {} ".format(rc3_radius,radii))
                        updated.write("{}       {}      {}      {}      {} \n".format(rc3_ra,rc3_dec,new_ra,new_dec,radii))
                        self.mosaic_all_bands(new_ra,new_dec,margin,radii,pgc)
                        return [float(new_ra),float(new_dec),margin,radii,pgc] 
                        # margin was already set as 6*rc3_radius during initial_run
                        # all additional mosaicking steps shoudl be 1.5 times this 
                else: #radii =@ if all SExtracted radius is <15 
                    print ("No detected RC3 sources in image. Mosaic using a larger margin")
                    # original automated mosaic program default 6*radius
                    # call on mosaic program with +50% original margin
                    r_mosaic_filename = self.mosaic_band('r',rc3_ra,rc3_dec,1.5*margin,rc3_radius,pgc)
                    self.source_info(r_mosaic_filename)
                    return ['@','@',1.5*margin,'@','@']
            else : 
                no_detection = open("../no_detected_rc3_candidate_nearby.txt",'a') # 'a' for append #'w')
                no_detection.write("rc3_ra       rc3_dec        rc3_radius        pgc \n")
                no_detection.write("{}       {}        {}        {} \n".format(self.rc3_ra,self.rc3_dec,self.rc3_radius,self.pgc))
        except (IOError):
            print ("File Not Found Error, if rfits is not found then mosaic an rfits")
            self.mosaic_band('r',self.rc3_ra,self.rc3_dec,3*self.rc3_radius,self.rc3_radius,self.pgc)
        except:
            print("Something went wrong when mosaicing PGC{}, just ignore it and keep mosaicing the next galaxy".format(str(pgc)))
            error = open ("sourceinfo_error.txt","a")
            error.write("{}       {}        {}        {} \n".format(self.rc3_ra,self.rc3_dec,self.rc3_radius,self.pgc))
            return['x','x','x','x','x']
Example #31
0
def run(img, outcat, type, limits=None):
    import os, sys, anydbm, time

    print img, outcat, type

    if type == 'star': mag_type = 'psf'
    if type == 'galaxy': mag_type = 'petro'

    print img
    os.system("rm outim")
    os.system('rm ' + outcat)
    os.system('rm sdss_out')

    if limits is not None:
        ramin = limits['ramin']
        ramax = limits['ramax']
        decmin = limits['decmin']
        decmax = limits['decmax']
    else:
        import commands, string
        command = 'dfits ' + img + ' | fitsort -d CD2_1'
        print command
        print commands.getoutput(command)
        if string.find(commands.getoutput(command), 'KEY') == -1:
            imcom = "dfits " + img + " | fitsort CRPIX1 CRPIX2 CRVAL1 CRVAL2 CD2_1 CD1_2 CD2_2 CD1_1 > ./outim"
        else:
            imcom = "dfits " + img + " | fitsort CRPIX1 CRPIX2 CRVAL1 CRVAL2 CDELT1 CDELT2 > ./outim"

        print imcom
        os.system(imcom)
        import re
        print open('outim', 'r').readlines()
        com = re.split('\s+', open("outim", 'r').readlines()[1][:-1])
        print com
        crpix1 = float(com[1])
        crpix2 = float(com[2])
        crval1 = float(com[3])
        crval2 = float(com[4])

        if string.find(commands.getoutput(command), 'KEY') == -1:
            cdelt1A = float(com[5])
            cdelt2A = float(com[6])
            cdelt1B = float(com[7])
            cdelt2B = float(com[8])

            if float(cdelt1A) != 0:
                cdelt1 = cdelt1A
                cdelt2 = cdelt2A
            else:
                cdelt1 = cdelt1B
                cdelt2 = cdelt2B
        else:
            cdelt1 = float(com[5])
            cdelt2 = float(com[6])

        print crpix1, crval1, cdelt1
        #ramin = crval1 - crpix1*cdelt1

        ramin = crval1 - 9000 * abs(cdelt1)
        print ramin
        ramax = crval1 + 9000 * abs(cdelt1)
        if ramax < ramin:
            top = ramin
            ramin = ramax
            ramax = top

        decmin = crval2 - 9000 * abs(cdelt2)
        decmax = crval2 + 9000 * abs(cdelt2)

    import sqlcl
    #lines = sqlcl.query("select ra,dec,u,g,r,i,z from star").readlines()

    #flags =  reduce(lambda x,y: x + ' AND ' + y, ["   ((flags_" + color + " & 0x10000000) != 0) \
    #        AND ((flags_" + color + " & 0x8100000800a4) = 0) \
    #    AND (((flags_" + color + " & 0x400000000000) = 0) or (psfmagerr_" + color + " <= 0.2)) \
    #       AND (((flags_" + color + " & 0x100000000000) = 0) or (flags_" + color + " & 0x1000) = 0) \
    # AND (flags_" + color + " & dbo.fPhotoFlags('BLENDED') = 0) " for color in ['u','g','r','i','z']])

    if type == 'star':
        flags = '\n\
               ((flags & 0x10000000) != 0)       \n\
AND ((flags & 0x8100000c00a4) = 0)     \n\
AND (((flags & 0x400000000000) = 0) or (psfmagerr_g <= 0.2)) \n\
AND (((flags & 0x100000000000) = 0) or (flags & 0x1000) = 0)    \n'

    elif type == 'galaxy':
        flags = '\n\
            ((flags & 0x10000000) != 0)     \n\
AND ((flags & 0x8100000c00a0) = 0)     \n\
AND (((flags & 0x400000000000) = 0) or (psfmagerr_g <= 0.2)) \n\
AND (((flags & 0x100000000000) = 0) or (flags & 0x1000) = 0) \n'

    query = "select clean, ra,dec,raErr,decErr," + mag_type + "Mag_u," + mag_type + "Mag_g," + mag_type + "Mag_r," + mag_type + "Mag_i," + mag_type + "Mag_z," + mag_type + "MagErr_u," + mag_type + "MagErr_g," + mag_type + "MagErr_r," + mag_type + "MagErr_i," + mag_type + "MagErr_z, flags from " + type + " where   ra between " + str(
        ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(
            decmin)[:8] + " and " + str(
                decmax)[:8] + " AND clean=1 and " + flags

    #query = "select clean, ra,dec,raErr,decErr," + mag_type + "Mag_u," + mag_type + "Mag_g," + mag_type + "Mag_r," + mag_type + "Mag_i," + mag_type + "Mag_z," + mag_type + "MagErr_u," + mag_type + "MagErr_g," + mag_type + "MagErr_r," + mag_type + "MagErr_i," + mag_type + "MagErr_z,flags_u,flags_g,flags_r,flags_i,flags_z, flags from " + type + " where   ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  + " AND flags & dbo.fPhotoFlags('BLENDED') = 0 "# AND   " + flags

    #query = "select ra,dec,raErr,decErr,objID, petroMag_u,petroMag_g,petroMag_r,petroMag_i,petroMag_z,petroMagErr_u,petroMagErr_g,petroMagErr_r,petroMagErr_i,petroMagErr_z,flags_u,flags_g,flags_r,flags_i,flags_z, flags from galaxy where   ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  #+ " AND flags & dbo.fPhotoFlags('BLENDED') = 0 "# AND   " + flags

    #query = "select top 10 flags_u, flags2_u from star where ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  + " "\

    #query = "select top 10 petroMagu, petroMagg, petroMagr, petroMagi, petroMagz, from star where ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  + " "
    print query

    lines = sqlcl.query(query).readlines()
    uu = open('store', 'w')
    import pickle
    pickle.dump(lines, uu)

    #import pickle
    #f=open('store','r')
    #m=pickle.Unpickler(f)
    #lines=m.load()

    columns = lines[0][:-1].split(',')
    #print columns
    data = []
    #print columns
    #print lines
    if lines[0][0:2] == 'No':
        return False, None

    for line in range(1, len(lines[1:]) + 1):
        #print lines[line]
        dt0 = {}
        for j in range(len(lines[line][:-1].split(','))):
            dt0[columns[j]] = lines[line][:-1].split(',')[j]

        import string
        if string.find(lines[line][:-1], 'font') == -1:
            data.append(dt0)
        #if string.find(lines[line][:-1],'font') != -1:
        #print lines[line][:-1]

    print len(data)
    print len(data[0])

    outwrite = open('sdss_out', 'w')
    print len(data)

    keys = [
        'SeqNr', ['dec', 'Dec'], ['ra', 'Ra'], 'raErr', 'decErr', 'umag',
        'gmag', 'rmag', 'imag', 'Bmag', 'Vmag', 'Rmag', 'Imag', 'zmag', 'uerr',
        'gerr', 'rerr', 'ierr', 'Berr', 'Verr', 'Rerr', 'Ierr', 'zerr', 'umg',
        'gmr', 'rmi', 'imz', 'BmV', 'VmR', 'RmI', 'Imz', 'umgerr', 'gmrerr',
        'rmierr', 'imzerr', 'BmVerr', 'VmRerr', 'RmIerr', 'Imzerr', 'A_WCS',
        'B_WCS', 'THETAWCS', 'Flag', 'Clean', ['ra', 'ALPHA_J2000'],
        ['dec', 'DELTA_J2000']
    ]

    #keys = ['SeqNr',['dec','Dec'],['ra','Ra'],'raErr','decErr','umag','gmag','rmag','imag','Bmag','Vmag','Rmag','Imag','zmag','uerr','gerr','rerr','ierr','Berr','Verr','Rerr','Ierr','zerr','umg','gmr','rmi','imz','BmV','VmR','RmI','Imz','umgerr','gmrerr','rmierr','imzerr','BmVerr','VmRerr','RmIerr','Imzerr','flags_u','flags_g','flags_r','flags_i','flags_z','A_WCS','B_WCS','THETAWCS','Flag','Clean',['ra','ALPHA_J2000'],['dec','DELTA_J2000']]
    seqnr = 1
    for els in range(len(data)):
        clean = data[els]['clean']
        if 1 == 1:  #int(flag)==1 :# 1==1: #data[els].has_key('u'):

            import math
            #print data[els].keys()

            ab_correction = {
                'u': -0.036,
                'g': 0.012,
                'r': 0.010,
                'i': 0.028,
                'z': 0.040
            }

            u = convert_to_pogson(float(data[els][mag_type + 'Mag_u']),
                                  'u') + ab_correction['u']
            g = convert_to_pogson(float(data[els][mag_type + 'Mag_g']),
                                  'g') + ab_correction['g']
            r = convert_to_pogson(float(data[els][mag_type + 'Mag_r']),
                                  'r') + ab_correction['r']
            i = convert_to_pogson(float(data[els][mag_type + 'Mag_i']),
                                  'i') + ab_correction['i']
            z = convert_to_pogson(float(data[els][mag_type + 'Mag_z']),
                                  'z') + ab_correction['z']

            uerr = float(data[els][mag_type + 'MagErr_u'])
            gerr = float(data[els][mag_type + 'MagErr_g'])
            rerr = float(data[els][mag_type + 'MagErr_r'])
            ierr = float(data[els][mag_type + 'MagErr_i'])
            zerr = float(data[els][mag_type + 'MagErr_z'])

            data[els]['Bmag'] = u - 0.8116 * (u -
                                              g) + 0.1313  #  sigma = 0.0095
            data[els]['Berr'] = math.sqrt((uerr * 0.19)**2. +
                                          (0.8119 * gerr)**2.)
            #B = g + 0.3130*(g - r) + 0.2271#  sigma = 0.0107

            #V = g - 0.2906*(u - g) + 0.0885#  sigma = 0.0129
            data[els]['Vmag'] = g - 0.5784 * (g -
                                              r) - 0.0038  #  sigma = 0.0054
            data[els]['Verr'] = math.sqrt((gerr * 0.42)**2. +
                                          (0.57 * rerr)**2.)

            #R = r - 0.1837*(g - r) - 0.0971#  sigma = 0.0106
            data[els]['Rmag'] = r - 0.2936 * (r -
                                              i) - 0.1439  #  sigma = 0.0072
            data[els]['Rerr'] = math.sqrt((rerr * 0.71)**2. +
                                          (0.29 * ierr)**2.)

            data[els]['Imag'] = r - 1.2444 * (r -
                                              i) - 0.3820  #  sigma = 0.0078
            data[els]['Ierr'] = math.sqrt((rerr * 0.24)**2. +
                                          (1.244 * ierr)**2.)
            #I = i - 0.3780*(i - z)  -0.3974#  sigma = 0.0063

            data[els]['umag'] = u
            data[els]['gmag'] = g
            data[els]['rmag'] = r
            data[els]['imag'] = i
            data[els]['zmag'] = z

            data[els]['umg'] = data[els]['umag'] - data[els]['gmag']
            data[els]['gmr'] = data[els]['gmag'] - data[els]['rmag']
            data[els]['rmi'] = data[els]['rmag'] - data[els]['imag']
            data[els]['imz'] = data[els]['imag'] - data[els]['zmag']

            data[els]['uerr'] = uerr
            data[els]['gerr'] = gerr
            data[els]['rerr'] = rerr
            data[els]['ierr'] = ierr
            data[els]['zerr'] = zerr

            data[els]['umgerr'] = math.sqrt(data[els]['uerr']**2. +
                                            data[els]['gerr']**2.)
            data[els]['gmrerr'] = math.sqrt(data[els]['gerr']**2. +
                                            data[els]['rerr']**2.)
            data[els]['rmierr'] = math.sqrt(data[els]['rerr']**2. +
                                            data[els]['ierr']**2.)
            data[els]['imzerr'] = math.sqrt(data[els]['ierr']**2. +
                                            data[els]['zerr']**2.)

            data[els]['BmV'] = data[els]['Bmag'] - data[els]['Vmag']
            data[els]['VmR'] = data[els]['Vmag'] - data[els]['Rmag']
            data[els]['RmI'] = data[els]['Rmag'] - data[els]['Imag']
            data[els]['Imz'] = data[els]['Imag'] - data[els]['zmag']

            data[els]['BmVerr'] = math.sqrt(data[els]['Berr']**2. +
                                            data[els]['Verr']**2.)
            data[els]['VmRerr'] = math.sqrt(data[els]['Verr']**2. +
                                            data[els]['Rerr']**2.)
            data[els]['RmIerr'] = math.sqrt(data[els]['Rerr']**2. +
                                            data[els]['Ierr']**2.)
            data[els]['Imzerr'] = math.sqrt(data[els]['Ierr']**2. +
                                            data[els]['zerr']**2.)

            #error = (float(data[els]['rowcErr_r'])**2. + float(data[els]['colcErr_r'])**2.)**0.5*0.4/3600.
            #if error < 0.0004: error=0.0004
            data[els]['A_WCS'] = 0.0004  #error #data[els]['Err'] #'0.0004'
            data[els]['B_WCS'] = 0.0004  #error #data[els]['decErr'] #'0.0004'
            data[els]['THETAWCS'] = '0'
            data[els]['Clean'] = str(clean)
            data[els]['Flag'] = '0'  #str(clean)

            seqnr += 1
            data[els]['SeqNr'] = seqnr

            lineh = ""
            #print data[els]
            if 1 == 1:  # data[els]['clean'] == 0:  #inspect_flags([data[els]['flags_u'],data[els]['flags_g'],data[els]['flags_r'],data[els]['flags_i'],data[els]['flags_z']],[data[els]['flags2_u'],data[els]['flags2_g'],data[els]['flags2_r'],data[els]['flags2_i'],data[els]['flags2_z']]):

                #print keys
                for key in keys:

                    if len(key) == 2:
                        key_dict = key[0]
                        key = key[1]
                    else:
                        key_dict = key
                    if (key == 'SeqNr' or key_dict == 'ra' or key_dict == 'dec'
                            or key[0:3] == 'Fla'):
                        num = '%(s)s' % {'s': str(data[els][key_dict])}
                    else:
                        num = '%(num).4f' % {'num': float(data[els][key_dict])}
                        num = '%s' % num
                    num.strip()
                    #elif key[0:2] != 'ra' and key[0:3] != 'dec':
                    #yy = ''
                    #for y in range(128):
                    #	if y < len(str(data[els][key])):
                    #		yy = yy + str(data[els][key])[y]
                    #	else:
                    #		yy = yy + ' '
                    #num = yy
                    #else: num = str(data[els][key])
                    lineh = lineh + num + " "
        #print lineh
                outwrite.write(lineh + "\n")
    outwrite.close()

    #lineh= "lc -C -B "
    #for key in data[els].keys():
    #	lineh = lineh + " -N '1 1 " + str(key) + "' "
    #lineh = lineh + " < outwrite > outf.cat"
    #print lineh
    #os.system(lineh)

    asc = open('asctoldac_sdss.conf', 'w')
    asc.write('VERBOSE = DEBUG\n')
    for column in keys:
        if len(column) == 2:
            name = column[1]
        else:
            name = column
        if column == 'objID' or column[0:3] == 'fla':
            type = 'STRING'
            htype = 'STRING'
            depth = '128'
        elif column == 'Flag':
            type = 'SHORT'
            htype = 'INT'
            depth = '1'
        elif column == 'SeqNr':
            type = 'LONG'
            htype = 'INT'
            depth = '1'
        elif len(column) == 2:  #column == 'Ra' or column == 'Dec':
            type = 'DOUBLE'
            htype = 'FLOAT'
            depth = '1'
        else:
            type = 'FLOAT'
            htype = 'FLOAT'
            depth = '1'
        asc.write('#\nCOL_NAME = ' + name + '\nCOL_TTYPE= ' + type +
                  '\nCOL_HTYPE= ' + htype +
                  '\nCOL_COMM= ""\nCOL_UNIT= ""\nCOL_DEPTH= ' + depth + '\n')

    asc.close()

    command = "asctoldac -i sdss_out -c asctoldac_sdss.conf -t STDTAB -o " + outcat
    os.system(command)
    print command

    if len(data) > 10:
        cov = True
    else:
        cov = False
    return cov, outcat
Example #32
0



query = "select ra,dec,z, zConf from SpecObj where   ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  + " AND z < 1.5 AND zConf > 0.90 " # AND flags & dbo.fPhotoFlags('BLENDED') = 0 "# AND   " + flags 


#query = "select clean, ra,dec,raErr,decErr,objID,rowcErr_u,colcErr_u,rowcErr_g,colcErr_g,rowcErr_r,colcErr_r,rowcErr_i,colcErr_i,rowcErr_z,colcErr_z,psfMag_u,psfMag_g,psfMag_r,psfMag_i,psfMag_z,psfMagErr_u,psfMagErr_g,psfMagErr_r,psfMagErr_i,psfMagErr_z,flags_u,flags_g,flags_r,flags_i,flags_z, flags from star where   ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  + " AND flags & dbo.fPhotoFlags('BLENDED') = 0 "# AND   " + flags 

print query
#query = "select top 10 flags_u, flags2_u from star where ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  + " "\

#query = "select top 10 psfMagu, psfMagg, psfMagr, psfMagi, psfMagz, from star where ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  + " "
print query

lines = sqlcl.query(query).readlines()
uu = open('store','w')
import pickle
pickle.dump(lines,uu)

#import pickle
#f=open('store','r')
#m=pickle.Unpickler(f)
#lines=m.load()


#raw_input()
columns = lines[0][:-1].split(',')
print columns
data = []
print columns 
Example #33
0
import sqlcl as s
import time

curT = lambda: int(round(time.time() * 1000))

times1 = []
times2 = []

num = 30

for i in range(num):
  print "starting test " + `i`

  t1 = curT()
  r1 = s.query("SELECT objID, ra, dec  FROM PhotoPrimary WHERE ((ra BETWEEN 331.04711618888882 AND 331.04767158888887) AND (dec BETWEEN 6.2921503555555551 AND 6.2927057555555548))").read()
  t11 = curT()
  times1.append(t11-t1)
  
  t2 = curT()
  r2 = s.query("SELECT p.objid, p.ra, p.dec FROM fGetNearbyObjEq(331.047393889,6.29242805556,0.02) n, PhotoPrimary p WHERE n.objID=p.objID").read()
  t22 = curT()
  times2.append(t22-t2)
  
avg1 = sum(times1)/num
avg2 = sum(times2)/num

print "Results for 1"
print times1
print "average: " + `avg1`
print '\n'
print "Results for 2"
Example #34
0
def get_sdss_spectra(umg, imz, gmr, rmi, number=4, tol=0.01, S_N=5):
    import sqlcl

    dict_names = [
        'plate', 'MJD', 'fiberID', 'ra', 'dec', 'mag_0', 'mag_1', 'mag_2'
    ]
    query = 'select top ' + str(number) + ' ' + reduce(
        lambda x, y: x + ',' + y, ['s.' + x for x in dict_names]
    ) + ' from specobjall as s join specphotoall as p on s.specobjid = p.specobjid where abs(s.mag_0 - s.mag_1 - ' + str(
        gmr
    ) + ') < ' + str(tol) + ' and abs(s.mag_1 - s.mag_2 - ' + str(
        rmi
    ) + ') < ' + str(tol) + ' and abs(s.mag_0 - s.mag_2 - ' + str(
        gmr + rmi
    ) + ') < ' + str(tol) + ' and s.sn_0 > ' + str(
        S_N
    ) + ' and s.sn_1 > ' + str(S_N) + ' and s.sn_2 > ' + str(
        S_N
    ) + ' and abs(s.mag_0 - s.mag_1 - (p.fibermag_g - p.fibermag_r)) < 0.1 and abs(s.mag_1 - s.mag_2 - (p.fibermag_r - p.fibermag_i)) < 0.1  order by -1.*s.sn_1'

    if rmi < 0.7:
        pattern = 'zbelodiesptype like "%v%" and zbelodiesptype not like "%var%"'
        #elif 0.7 < rmi < 1.0: pattern = '(zbelodiesptype like "%G%v%" or zbelodiesptype like "%K%v%" or zbelodiesptype like "%M%v%")'
    else:
        pattern = 'zbelodiesptype like "%M%v%"'

    query = 'select top ' + str(number) + ' ' + reduce(
        lambda x, y: x + ',' + y, ['s.' + x for x in dict_names]
    ) + ' from specobjall as s join specphoto as p on s.specobjid = p.specobjid join sppParams sp on sp.specobjid = s.specobjid where zbclass="STAR" and ' + pattern + ' and abs(s.mag_0 - s.mag_1 - ' + str(
        gmr
    ) + ') < ' + str(tol) + ' and abs(s.mag_1 - s.mag_2 - ' + str(
        rmi
    ) + ') < ' + str(tol) + ' and abs(s.mag_0 - s.mag_2 - ' + str(
        gmr + rmi
    ) + ') < ' + str(tol) + ' and s.sn_0 > ' + str(
        S_N
    ) + ' and s.sn_1 > ' + str(S_N) + ' and s.sn_2 > ' + str(
        S_N
    ) + ' and abs(s.mag_0 - s.mag_1 - (p.fibermag_g - p.fibermag_r)) < 0.1 and abs(s.mag_1 - s.mag_2 - (p.fibermag_r - p.fibermag_i)) < 0.1  and abs(' + str(
        umg) + ' - (p.psfMag_u - p.psfMag_g)) < 0.05 and abs(' + str(
            imz) + ' - (p.psfMag_i - p.psfMag_z)) < 0.05 \
order by -1.*s.sn_1'

    query = 'select top ' + str(number) + ' ' + reduce(
        lambda x, y: x + ',' + y, ['s.' + x for x in dict_names]
    ) + ' from specobjall as s join specphoto as p on s.specobjid = p.specobjid join sppParams sp on sp.specobjid = s.specobjid where zbclass="STAR" and ' + pattern + ' and abs(s.mag_0 - s.mag_1 - ' + str(
        gmr
    ) + ') < ' + str(tol) + ' and abs(s.mag_1 - s.mag_2 - ' + str(
        rmi
    ) + ') < ' + str(tol) + ' and abs(s.mag_0 - s.mag_2 - ' + str(
        gmr + rmi
    ) + ') < ' + str(tol) + ' and s.sn_0 > ' + str(
        S_N
    ) + ' and s.sn_1 > ' + str(S_N) + ' and s.sn_2 > ' + str(
        S_N
    ) + ' and abs(s.mag_0 - s.mag_1 - (p.fibermag_g - p.fibermag_r)) < 0.1 and abs(s.mag_1 - s.mag_2 - (p.fibermag_r - p.fibermag_i)) < 0.1  and abs(' + str(
        umg) + ' - (p.psfMag_u - p.psfMag_g)) < 0.05 and abs(' + str(
            imz) + ' - (p.psfMag_i - p.psfMag_z)) < 0.05 \
order by -1.*s.sn_1'

    import time
    time.sleep(1.5)

    print query
    lines = sqlcl.query(query).readlines()
    print lines

    dicts = []

    if lines[0] != 'N':

        for line in lines[1:]:
            dict = {}
            line = line.replace('\n', '')
            import re
            res = re.split(',', line)
            print res
            for i in range(len(res)):
                if dict_names[i] == 'fiberID' or dict_names[
                        i] == 'plate' or dict_names[i] == 'MJD':
                    dict[dict_names[i]] = int(res[i])
                else:
                    dict[dict_names[i]] = (res[i])
            print dict
            dicts.append(dict)

        print dicts

    return dicts
Example #35
0
import string

infile_dir = sys.argv[1]
oufile_dir = sys.argv[2]

infile_list = os.listdir(infile_dir)
url = sqlcl.default_url
fmt = 'csv'

for f in infile_list:
    infile = './' + os.path.join(infile_dir, f)
    oufile = './' + os.path.join(oufile_dir, f) + '.' + fmt

    print "request %s ..." % infile
    qry = open(infile).read()
    file = sqlcl.query(qry, url, fmt)
    line = file.readline()

    print "save %s to %s..." % (infile, oufile)
    oufile_fd = open(oufile, 'w')
    sqlcl.write_header(oufile_fd, "#", url, qry)

    if line.startswith('ERROR'):
        oufile_fd.write('ERROR')
        oufile_fd.close()
        continue

    while line:
        oufile_fd.write(string.rstrip(line) + os.linesep)
        line = file.readline()
Example #36
0
import sqlcl
import sys, os
from csvMathMod import csv2math

# Path for transfer files
prefix = "/ram/"
# prefix for transfer files
xfer = "xfer-"

if len(sys.argv) != 2:
    print('Syntax is : python sdss_query.py "SQL query"')
    exit(0)
print("Executing Query : " + sys.argv[1] + "\n")

lines = sqlcl.query(sys.argv[1]).readlines()
# New versions seem to produce an extra first line with Table so remove
print lines[1:]
os.system("rm " + prefix + xfer + "* 2> /dev/null")
csv2math(lines[1:])
Example #37
0
def get_SDSS(ra0, dec0, rad=1/60., name='', silent=False, debug=False):
    '''
    >> data = get_SDSS(ra, dec, rad=1, dir='./' name='mydata')

    submit CAS job via sqlcl
    input:
     ra, dec (deg), can be arrays
    optional input:
      radius=1/60. (deg)
      name if given, we write file name
      silent=False shut it.
    note:
     slow for many object because we loop over input coords; 
     this could be log(N) faster if I knew how 
     to upload coordinates and run fgetNearByObjEq on this list. 

    '''
    if np.isscalar(ra0):
        ra0 = [ra0]
        dec0 = [dec0]

    out_list = [] 
    for ra, dec in zip(ra0, dec0):
        cas = SDSS_cas.replace('__RA__',str(ra)).replace('__DEC__', str(dec)).replace('__RAD__',str(rad*60))
    
        if not(silent):
            print 'running CASjob:\n',cas

        result = sqlcl.query(cas)

        if not(silent):
            print 'CAS job done, now reading query...'

        #print result.readlines()
        lines = result.readlines()

        if debug:
            print ra, dec, lines

        if len(lines)<=2:
            print 'no sources found, for ra,dec:', ra, dec
        else:
            data = readascii(lines=lines[2:], names=lines[1].split(','), delimiter=',')
            out_list.append(data)

    if len(out_list)==0:
        return None # no data, with one input

    if len(out_list)==1:
        out = out_list[0]

    if len(out_list)>1:
        data_arr = np.repeat(out_list[0],1)
        for dd in out_list[1:]:
            data_arr = rec.merge_rec(data_arr, np.repeat(dd,1))
            #data_arr =  np.concatenate(data_arr, dd)
        out= data_arr

    if name: 
        if not(silent):
            print '# of entries:', len(out)
            print 'writing to ', name
        pyfits.writeto(name, out, clobber=True)
    
    return out
Example #38
0
def getimg(ira,
           idec,
           imsize,
           BW=False,
           DSS=None,
           fullname=False,
           slitangle="Parallactic"):
    ''' Grab an SDSS image from the given URL, if possible

    Parameters:
    ----------
    ira: (float or Quantity) RA in decimal degrees
    idec: (float or Quantity) DEC in decimal degrees
    '''

    # Strip units as need be
    try:
        ra = ira.value
    except KeyError:
        ra = ira
        dec = idec
    except AttributeError:
        ra = ira
        dec = idec
    else:
        dec = idec.value

    # Get URL
    if DSS == None:  # Default
        url = sdsshttp(ra, dec, imsize)
    else:
        url = dsshttp(ra, dec, imsize)  # DSS

    # Request
    rtv = requests.get(url)

    # Commenting out the next section for now because it crashes in SDSS.query_region and this does not affect finder chart generation
    # Query for photometry
    #    coord = SkyCoord(ra=ra*u.degree, dec=dec*u.degree)
    #    phot = SDSS.query_region(coord, radius=0.02*u.deg)
    #print phot

    #   if phot is None:
    #       print('getimg: Pulling from DSS instead of SDSS')
    #       BW = 1
    #       url = dsshttp(ra,dec,imsize) # DSS
    #       rtv = requests.get(url)

    img = Image.open(StringIO(rtv.content))

    # B&W ?
    if BW:
        import PIL.ImageOps
        img2 = img.convert("L")
        img2 = PIL.ImageOps.invert(img2)
        img = img2

    # Find offset by submitting a query through sqlcl.py, saving it as all_offsets.da t, and searching through it
    with open('all_offsets.dat', 'w') as query:
        conditions = "SELECT r, ra, dec FROM Star WHERE ra BETWEEN " + str(
            ira - imsize / 120.0) + " AND " + str(
                ira + imsize / 120.0) + " AND dec BETWEEN " + str(
                    idec - imsize / 120.0) + " AND " + str(
                        idec + imsize / 120.0) + " and (r < 19) and (r > 7)"
        query_results = sqlcl.query(conditions).read()
        query.write(query_results)
    query.close()

    offsets = Table.read('all_offsets.dat',
                         format='ascii',
                         names=('r_mag', 'RA', 'DEC'))
    #    xdb.set_trace()
    #    min_distance = 0.4
    #       if abs(ra-off_ra) <= 0.035 and abs(dec-off_dec) <= 0.035:
    #           distance=(((ra-off_ra)**2)+((dec-off_dec)**2))**(1./2.)
    #           if distance < min_distance:
    #               min_distance = distance
    #               min_r_mag = off_r_mag
    #               min_off_ra = off_ra
    #               min_off_dec = off_dec

    min_r_mag = 19

    if len(offsets) == 0:
        print "No offsets found within +/-0.03 in RA and DEC with r_mag <", min_r_mag

    if len(offsets) >= 1:
        for j in range(0, len(offsets)):
            off_r_mag = offsets['r_mag'][j]
            off_ra = offsets['RA'][j]
            off_dec = offsets['DEC'][j]

            if off_r_mag <= min_r_mag:
                min_r_mag = off_r_mag
                min_off_ra = off_ra
                min_off_dec = off_dec

        # Change object coordinates from decimal degrees to HMS/DMS
        string_ra = str(ira)
        string_dec = str(idec)
        hms_ra, dms_dec, other = calc_offset.decdeg_to_radec(
            string_ra, string_dec)

        # Change target coordinates from decimal degrees to HMS/DMS
        string_off_ra = str(min_off_ra)
        string_off_dec = str(min_off_dec)
        hms_off_ra, dms_off_dec, off_other = calc_offset.decdeg_to_radec(
            string_off_ra, string_off_dec)

        # Calculate offset move
        del_ra, e_or_w, del_dec, n_or_s = offset(ra, dec, min_off_ra,
                                                 min_off_dec)

        # Obtain a name for the file
        if fullname:
            outfil = "../finding_charts/"
            outfil += "J" + hms_ra[0] + hms_ra[1] + hms_ra[2].split(".")[0]
            outfil += "_"
            outfil += dms_dec[0].replace("-", "m").replace(
                "+", "p") + dms_dec[1] + dms_dec[2].split(".")[0]
            outfil += ".pdf"
        else:
            outfil = "J" + hms_ra[0] + hms_ra[1] + dms_dec[0] + dms_dec[
                1] + ".pdf"
        # Check if the file already exists
        if os.path.exists(outfil):
            ans = ""
            while (ans != "n") and (ans != "y"):
                ans = raw_input("\nFile already exists:\n" + outfil +
                                "\nOverwrite (y/n): ")
            if ans == "n":
                print "Finder not written!"
                return
        # Plot the figure
        fig = plt.figure(dpi=1200)
        fig.set_size_inches(8.0, 10.5)

        # Font
        plt.rcParams['font.family'] = 'times new roman'
        ax = plt.gca()

        # Image
        if BW == 1: cmm = cm.Greys_r
        else: cmm = None
        cradius = imsize / 30.
        plt.imshow(img,
                   cmap=cmm,
                   aspect='equal',
                   extent=(-imsize / 2., imsize / 2, -imsize / 2., imsize / 2))

        # Axes
        plt.xlim(-imsize / 2., imsize / 2.)
        plt.ylim(-imsize / 2., imsize / 2.)

        # Label
        plt.xlabel('Relative ArcMin', fontsize=20)
        xpos = 0.12 * imsize
        ypos = 0.02 * imsize
        plt.text(-imsize / 2. - xpos, 0., 'EAST', rotation=90., fontsize=20)
        plt.text(0.,
                 imsize / 2. + ypos,
                 'NORTH',
                 fontsize=20,
                 horizontalalignment='center')
        plt.text(0.,
                 -imsize / 2. - 8 * ypos,
                 'Slit Angle = ' + slitangle,
                 fontsize=20,
                 horizontalalignment='center')

        # Title
        plt.text(0.25,
                 1.28,
                 'Object Coordinates:',
                 fontsize=18,
                 horizontalalignment='center',
                 transform=ax.transAxes)
        plt.text(0.25,
                 1.23,
                 'RA = ' + str(hms_ra[0]) + ":" + str(hms_ra[1]) + ":" +
                 str(hms_ra[2]),
                 fontsize=20,
                 horizontalalignment='center',
                 transform=ax.transAxes)
        plt.text(0.25,
                 1.18,
                 'DEC = ' + str(dms_dec[0]) + ":" + str(dms_dec[1]) + ":" +
                 str(dms_dec[2]),
                 fontsize=20,
                 horizontalalignment='center',
                 transform=ax.transAxes)
        plt.text(0.75,
                 1.28,
                 'Offset Coordinates (r=' + str(format(min_r_mag, '.2f')) +
                 '):',
                 fontsize=18,
                 horizontalalignment='center',
                 transform=ax.transAxes)
        plt.text(0.75,
                 1.23,
                 'RA = ' + str(hms_off_ra[0]) + ":" + str(hms_off_ra[1]) +
                 ":" + str(hms_off_ra[2]),
                 fontsize=20,
                 horizontalalignment='center',
                 transform=ax.transAxes)
        plt.text(0.75,
                 1.18,
                 'DEC = ' + str(dms_off_dec[0]) + ":" + str(dms_off_dec[1]) +
                 ":" + str(dms_off_dec[2]),
                 fontsize=20,
                 horizontalalignment='center',
                 transform=ax.transAxes)
        plt.text(0.5,
                 1.13,
                 'From Offset, move to get to Object:',
                 fontsize=18,
                 horizontalalignment='center',
                 transform=ax.transAxes)
        plt.text(0.5,
                 1.08,
                 str(format(del_ra, '.2f')) + "'' " + e_or_w + " and " +
                 str(format(del_dec, '.2f')) + "'' " + n_or_s,
                 fontsize=20,
                 horizontalalignment='center',
                 transform=ax.transAxes)

        # Circle for target, then offset
        circle = plt.Circle((0, 0), cradius, color='y', fill=False)
        plt.gca().add_artist(circle)

        if e_or_w == 'W' and n_or_s == 'S':
            circle_offset = plt.Circle((-del_ra / 60, del_dec / 60),
                                       cradius,
                                       color='g',
                                       fill=False)
            plt.gca().add_artist(circle_offset)
        elif e_or_w == 'W' and n_or_s == 'N':
            circle_offset = plt.Circle((-del_ra / 60, -del_dec / 60),
                                       cradius,
                                       color='g',
                                       fill=False)
            plt.gca().add_artist(circle_offset)
        elif e_or_w == 'E' and n_or_s == 'S':
            circle_offset = plt.Circle((del_ra / 60, del_dec / 60),
                                       cradius,
                                       color='g',
                                       fill=False)
            plt.gca().add_artist(circle_offset)
        elif e_or_w == 'E' and n_or_s == 'N':
            circle_offset = plt.Circle((del_ra / 60, -del_dec / 60),
                                       cradius,
                                       color='g',
                                       fill=False)
            plt.gca().add_artist(circle_offset)

        # Spectrum??
        show_spec = False
        if show_spec:
            spec_img = xgs.get_spec_img(ra_tab['RA'][qq], ra_tab['DEC'][qq])
            plt.imshow(spec_img,
                       extent=(-imsize / 2.1, imsize * (-0.1), -imsize / 2.1,
                               imsize * (-0.2)))

        # Write
        if show_spec:
            plt.savefig(outfil, dpi=300)
        else:
            plt.savefig(outfil)
        print 'finder: Wrote ' + outfil
Example #39
0
def crossID(ra, dec, unit=None, dr=2., fields=None):
    """
    Perform object cross-ID in SDSS using SQL.
    
    Search for objects near position (ra, dec) within some radius using
    Tamas Budavari's SQL tool (sqlcl.py).
    
    Parameters
    ----------
    ra : float, int, str, tuple
        An object that represents a right ascension angle.
    dec : float, int, str, tuple
        An object that represents a declination angle.
    unit : `~astropy.units.UnitBase`, str
        The unit of the value specified for the angle
    dr : int, float
        Radius of region to perform object cross-ID (arcseconds).
    fields : list, optional
        SDSS PhotoObj or SpecObj quantities to return. If None, defaults
        to quantities required to find corresponding spectra and images
        of matched objects (e.g. plate, fiberID, mjd, etc.).
             
    See documentation for astropy.coordinates.angles for more information 
    about ('ra', 'dec', 'unit') parameters.
    
    Examples
    --------
    xid = sdss.crossID(ra='0h8m05.63s', dec='14d50m23.3s')
    
    for match in xid:
        print match['ra'], match['dec'], match['objid']

    Returns
    -------
    List of all objects found within search radius. Each element of list is 
    a dictionary containing information about each matched object.
    """
    
    if not isinstance(ra, coord.angles.RA):
        ra = coord.RA(ra, unit=unit)
    if not isinstance(ra, coord.angles.Dec):    
        dec = coord.Dec(dec, unit=unit)
    
    if fields is None:
        fields = photoobj_defs + specobj_defs
        
    # Convert arcseconds to degrees
    dr /= 3600.    
            
    Nfields = len(fields)    
        
    q_select = 'SELECT '
    for field in fields:
        if field in photoobj_defs:
            q_select += 'p.%s,' % field
        if field in specobj_defs:
            q_select += 's.%s,' % field
    q_select = q_select.rstrip(',')
    q_select += ' '
    
    q_from = 'FROM PhotoObjAll AS p '
    q_join = 'JOIN SpecObjAll s ON p.objID = s.bestObjID '
    q_where = 'WHERE (p.ra between %g and %g) and (p.dec between %g and %g)' \
        % (ra.degrees-dr, ra.degrees+dr, dec.degrees-dr, dec.degrees+dr)
    
    q = sqlcl.query("%s%s%s%s" % (q_select, q_from, q_join, q_where))
    
    results = []
    cols = q.readline()
    while True:
        line = q.readline().replace('\n', '').split(',')
        
        if len(line) == 1:
            break
        
        tmp = {}
        for i, val in enumerate(line):
            
            field = fields[i]
            
            if val.isdigit(): 
                tmp[field] = int(val)
            else:
                try: 
                    tmp[field] = float(val)
                except ValueError: 
                    tmp[field] = str(val)
                    
        results.append(tmp)            

    return results
Example #40
0
    def getsdssphotcatv2(self):
        # going to split query into 3 separate calls.  Hopefully this will alleviate the timeout errors!
        # then can merge files with 'join'
        print 'Getting SDSS phot cat for ', self.prefix
        drsearch = self.dr * 60.  #search radius in arcmin for sdss query

        for k in range(3):
            if k == 0:
                #query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID, g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and  (g.PrimTarget & 0x00000040) > 0 and (g.specObjID = 0)" % (self.cra,self.cdec,drsearch)
                # removing PrimTarget constraint
                #query="select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID, g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and (g.specObjID = 0)" % (self.cra,self.cdec,drsearch)
                # adding r mag cut
                query = "select g.ra, g.dec, g.u, g.g, g.r, g.i, g.z, g.objid, g.specObjID, g.petroMag_u, g.petroMag_g, g.petroMag_r, g.petroMag_i, g.petroMag_z,g.petroRad_u, g.petroRad_g, g.petroRad_r, g.petroRad_i, g.petroRad_z, g.petroR50_u, g.petroR50_g, g.petroR50_r, g.petroR50_i, g.petroR50_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.r < 22 and g.objID = n.objID and (g.specObjID = 0)" % (
                    self.cra, self.cdec, drsearch)
            if k == 1:
                #query="select g.petroR90_u, g.petroR90_g, g.petroR90_r, g.petroR90_i, g.petroR90_z, g.isoA_r, g.isoB_r, g.isoPhi_r, g.isoPhiErr_r, g.deVRad_r, g.deVRadErr_r, g.deVPhi_r, g.deVPhiErr_r, g.deVMag_r, g.expRad_r, g.expRadErr_r, g.expAB_r, g.expABErr_r, g.expPhi_r, g.expPhiErr_r, g.expMag_r, g.expMagErr_r, g.extinction_u,g.extinction_g,g.extinction_r,g.extinction_i,g.extinction_z, g.dered_u, g.dered_g, g.dered_r, g.dered_i, g.dered_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and  (g.PrimTarget & 0x00000040) > 0 and (g.specObjID = 0)" % (self.cra,self.cdec,drsearch)
                query = "select g.petroR90_u, g.petroR90_g, g.petroR90_r, g.petroR90_i, g.petroR90_z, g.isoA_r, g.isoB_r, g.isoPhi_r, g.isoPhiErr_r, g.deVRad_r, g.deVRadErr_r, g.deVPhi_r, g.deVPhiErr_r, g.deVMag_r, g.expRad_r, g.expRadErr_r, g.expAB_r, g.expABErr_r, g.expPhi_r, g.expPhiErr_r, g.expMag_r, g.expMagErr_r, g.extinction_u,g.extinction_g,g.extinction_r,g.extinction_i,g.extinction_z, g.dered_u, g.dered_g, g.dered_r, g.dered_i, g.dered_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.r < 22  and g.objID = n.objID and (g.specObjID = 0)" % (
                    self.cra, self.cdec, drsearch)
            if k == 2:
                #query="select g.run, g.rerun, g.camcol, g.field, g.err_u,g.err_g,g.err_r,g.err_i,g.err_z,g.rowc_u, g.rowc_g, g.rowc_r,g.rowc_i,g.rowc_z,g.colc_u,g.colc_g,g.colc_r,g.colc_i,g.colc_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.objID = n.objID and  (g.PrimTarget & 0x00000040) > 0 and (g.specObjID = 0)" % (self.cra,self.cdec,drsearch)

                query = "select g.run, g.rerun, g.camcol, g.field, g.err_u,g.err_g,g.err_r,g.err_i,g.err_z,g.rowc_u, g.rowc_g, g.rowc_r,g.rowc_i,g.rowc_z,g.colc_u,g.colc_g,g.colc_r,g.colc_i,g.colc_z from galaxy g, dbo.fGetNearbyObjEq(%12.8f,%12.8f,%8.3f) n where g.r < 22 and g.objID = n.objID and (g.specObjID = 0)" % (
                    self.cra, self.cdec, drsearch)

    # part 1 of query
            part = k + 1
            print 'running part %i of query' % (part)
            flag = 0
            nrun = 1
            print 'getting to while loop'
            while flag == 0:
                print 'inside while loop'
                #changed so that only galaxies w/out spectra are returned
                print query
                start_time = time.time()
                try:
                    lines = sqlcl.query(query).readlines()
                except IOError:
                    print "IOError for cluster", self.prefix, " trying phot query again"
                    lines = sqlcl.query(query).readlines()

                elapsed_time = time.time() - start_time
                print 'time to execute query = ', elapsed_time, ' sec, ', elapsed_time / 60., ' min'
                print "got number+1 phot objects = ", len(lines)

                n = homedir + 'research/LocalClusters/SDSSCatalogs/' + str(
                    self.prefix) + 'galaxy.photcat.p' + str(part) + '.dat'
                outfile = open(n, 'w')
                j = 0
                flag = 1
                if (len(lines) > 1.):
                    for line in lines[1:]:
                        if j < 0:
                            print line
                            j = j + 1
                        outfile.write(line)
                        if line.find('Server.ScriptTimeout') > -1:
                            flag = 0
                        elif line.find('Timeout') > -1:
                            flag = 0
                        elif line.find('ERROR') > -1:
                            flag = 0
                        elif line.find('error') > -1:
                            flag = 0
                outfile.close()
                nrun += 1
                if nrun > 15:
                    break
                if flag == 0:
                    print self.prefix
                    print 'Running query again b/c of ScriptTimeout'
                    print 'starting attempt = ', nrun
from math import trunc
import sqlcl
#Must run in Python 2
DEBUG = True

ra= 4.33041666667
dec=6.72333333333
radius =  0.0162872870485/2. #radius = diameter/2
margin = 1.2* radius  #radius + radius * 0.2
 
#This is proper margin, but for multiple entries we are using margin =0.5 as test
#margin=0.2* radius + radius
# margin =0.5
if (DEBUG) : print ("Querying data that lies inside margin")
data =[]
result=sqlcl.query("SELECT distinct run,camcol,field FROM PhotoObj WHERE  ra between " + str(ra) +"-" +str(margin) +" and "+ str(ra) +"+"+ str(margin) +" and dec between "+ str(dec) + "-"+ str(margin)+ " and "+str(dec)+"+"+ str(margin)).readlines()
print result
count =0
for i in result:
    if count>1:
        list =i.split(',')
        list[2]= list[2][:-1]
        data.append(list)
    count +=1
if (DEBUG): 
	print ( "Complete Query. These data lies within margin: ")
	print (data)


bands=['u','g','r','i','z']
for ele in bands:
def sort_data():
    objid=[]
    extinction=[]
    teff=[]
    hbreqw=[]
    hbcont=[]
    hdflux=[]
    hdreqw=[]
    hdcont=[]
    objs=[]
    plate=[]
    fiber=[]
    mjd=[]
    n2flux=[] #n2 is now h-a, but both are not used
    n2reqw=[]
    magu=[]
    magg=[]
    magr=[]
    magi=[]
    magz=[]

    array1=[objid,extinction,teff,hbreqw,hbcont,hdflux,hdreqw,hdcont,objs,plate,fiber,mjd,\
            n2flux,n2reqw,magu,magg,magr,magi,magz]
    #and p.extinction_g between "+str(exta)+" AND "+str(extb)+ ##
    query = "SELECT p.objID, \
    p.extinction_g, s.elodieTEff, p.extinction_g, p.extinction_g, p.extinction_g, p.extinction_g, p.extinction_g, \
    p.obj, s.plate, s.fiberID, s.mjd, p.extinction_g, p.extinction_g, p.psfMag_u, \
    p.psfMag_g, p.psfMag_r, p.psfMag_i, p.psfMag_z \
    FROM PhotoObj AS p \
    JOIN SpecObj as s ON s.specobjID=p.specobjID \
    WHERE psfMag_r BETWEEN 15.0 and 19.0 \
    and p.type=6  \
     and  dbo.fPhotoStatus('PRIMARY')>0 and dbo.fPhotoFlags('STATIONARY')>0 \
    and calibStatus_r=1 \
    and s.elodieTEff!=0 and s.elodieFeH!=0 and s.elodieLogG!=0 \
    and ((flags&dbo.fPhotoFlags('BLENDED')) \
    +(flags&dbo.fPhotoFlags('DEBLEND_TOO_MANY_PEAKS')) + \
    (flags&dbo.fPhotoFlags('SATURATED')) \
    +(flags&dbo.fPhotoFlags('BADSKY'))+ \
    (flags&dbo.fPhotoFlags('COSMIC_RAY')) \
    +(flags&dbo.fPhotoFlags('PEAKS_TOO_CLOSE'))+ \
    (flags&dbo.fPhotoFlags('NOTCHECKED_CENTER')) \
    +(flags&dbo.fPhotoFlags('SATUR_CENTER'))+ \
    (flags&dbo.fPhotoFlags('INTERP_CENTER')) \
    +(flags&dbo.fPhotoFlags('INTERP'))+ \
    (flags&dbo.fPhotoFlags('PSF_FLUX_INTERP')))=0 \
    AND  (psfMag_u-psfmag_g) between 0.82-0.08 and 0.82+0.08 \
    AND (psfMag_g-psfmag_r) between 0.3-0.08 and 0.30+0.08 \
    AND (psfMag_r-psfmag_i) between 0.09-0.08 and 0.09+0.08 \
    AND (psfMag_i-psfmag_z) between 0.02-0.08 and 0.02+0.08 \
    ORDER BY p.extinction_g DESC"
    alldata=sqlcl.query(query).read()
    interim=alldata.replace("\n",",")
    nent=19 #(number of query columns)
    compiled=interim.split(",")

    #sort all values into arrays:
    for j in range(nent): 
        for i in range(j+nent,len(compiled)-1,nent):
            array1[j].append(float(compiled[i])) #note, plate/mjd/fiber should be int

    #some of the values should be type(int):
    for i in range(len(plate)): 
            plate[i]=int(plate[i])
            mjd[i]=int(mjd[i])
            fiber[i]=int(fiber[i])

    return plate, mjd, fiber, extinction, objid
import sqlcl
import numpy as np
with open("rc3_ra_dec_diameter_pgc.txt",'r') as f:
    for line in f:
        ra = float(line.split()[0])
        dec = float(line.split()[1])
        radius = float(line.split()[2])/2. #radius = diameter/2'
        margin = 3*radius
        pgc=str(line.split()[3]).replace(' ', '')
        clean=True
        filename = "{},{}".format(str(ra),str(dec))
        other_rc3s = sqlcl.query("SELECT distinct rc3.ra, rc3.dec FROM PhotoObj as po JOIN RC3 as rc3 ON rc3.objid = po.objid  WHERE po.ra between {0}-{1} and  {0}+{1} and po.dec between {2}-{3} and  {2}+{3}".format(str(ra),str(margin),str(dec),str(margin))).readlines()
        data =[]
        count =0
        for i in other_rc3s:
            if count>1:
                list =i.split(',')
                list[0] = float(list[0])
                list[1]= float(list[1][:-1])
                data.append(list)
            count += 1 
        print (data)
        if (len(data)>1 and len(data)%2==0):
            d2p= np.array(data[0])-np.array(data[1])
            print ("d2p: {}".format(d2p))
            # if negative then a is on the left of b
            # if positive then b is on the right of b
            # or something like that
            # then we do pairwise comparison to figure out their relative locations
        #Finding the difference between the 2 points
        # d2p = np.array()-np.array()
	sys.exit()

search_rad = str(3.0/60.0)
photo_objid = ""
optlist, args = getopt.getopt(sys.argv[1:], 'r:i:')
for o, a in optlist:
	if o == "-r":
		search_rad = str(float(a)/60.0)
	if o == "-i":
		photo_objid = a

if photo_objid == "" :
	ra = sys.argv[1]
	dec = sys.argv[2]
	sql_query = "select P.objid, P.modelMag_u, P.modelMag_g, P.modelMag_r, P.modelMag_i, P.modelMag_z from PhotoObjAll P, dbo.fGetNearbyObjAllEq("+ra+","+dec+","+search_rad+") n where P.objID = n.objID and P.specObjID > 0"
	query_result = sqlcl.query(sql_query).readlines()
	if len(query_result) > 1 :
		data_part = string.split(query_result[1],",")
		photo_objid = data_part[0]
		time.sleep(1.0)
		sql_query = "select S.SpecObjID, S.plate, S.fiberID, S.mjd, S.z, S.zErr, S.specClass from SpecObjAll S where S.bestObjID ="+photo_objid
		query_result = sqlcl.query(sql_query).readlines()
		if len(query_result) > 1 :
			temp = string.split(query_result[1], ",")
			output_str = ""
			for x in temp:
				output_str = output_str + x.strip() + " "
			print output_str
		else :
			print "No spectrum found but photometric objid = ", photo_objid
		time.sleep(1.0)
    def mosaic_band(self,band,ra,dec,margin,radius,pgc):#,clean=True):
        '''
        Input: source info param
        Create a mosaic fit file for the specified band.
        Return: String filename of resulting mosaic
        '''
        print ("------------------mosaic_band----------------------")
        DEBUG = True
        # output = open("rc3_galaxies_outside_SDSS_footprint.txt",'a') # 'a' for append #'w')
        # unclean = open("rc3_galaxies_unclean","a")
        # filename = "{},{}".format(str(ra),str(dec))
        filename = str(ra)+str(dec)
        #print (margin/radius)
        if (DEBUG) : print ("Querying data that lies inside margin")
        #result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  ra between {0}-{1} and  {0}+{1}and dec between {2}-{3} and  {2}+{3}".format(str(ra),str(margin),str(dec),str(margin))).readlines()
        result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
        clean_result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  CLEAN =1 and ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
        # clean_result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  CLEAN =1 and ra between {0}-{1} and  {0}+{1}and dec between {2}-{3} and  {2}+{3}".format(str(ra),str(margin),str(dec),str(margin))) .readlines()
        clean = True
        print (result)
        print (clean_result)
        if (result[0][5:]=="<html>"):
            print("strange error from SQL server")
            return -1
        if (result[1]=='error_message\n' or clean_result[1]=='error_message\n'):
    	    #Case where doing more than 60 queries in 1 minute
            time.sleep(60)
            #results are messed up, need to re-query
            result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
            clean_result = sqlcl.query( "SELECT distinct run,camcol,field FROM PhotoObj WHERE  CLEAN =1 and ra between "+str(ra)+"-"+str(margin)+" and " +str(ra)+"+"+str(margin)+"and dec between "+str(dec)+"-"+str(margin)+" and "+ str(dec)+"+"+str(margin)).readlines()
        if (len(result)!=len(clean_result) and band=='u'):
            #only print this once in the u band. If it is unclean in u band (ex. cosmic ray, bright star..etc) then it must be unclean in the other bands too.
            print ("Data contain unclean images")
            clean=False
            unclean.write(str(ra)+"     "+str(dec)+"     "+str(radius)+"     "+pgc)
            # unclean.write("{}     {}     {}     {} \n".format(str(ra),str(dec),str(radius),pgc))    
        data =[]
        count =0
        for i in result:
            if count>1:
                list =i.split(',')
                list[2]= list[2][:-1]
                data.append(list)
            count += 1 
        print (data)
        if (len(data)==0 and band=='r'): #you will only evounter non-footprint galaxy inint run , because after that we just take the footprint gaalxy already mosaiced (init) from rfits
            if (DEBUG): print ('The given ra, dec of this galaxy does not lie in the SDSS footprint. Onto the next galaxy!')#Exit Program.'
            output.write(str(ra)+ "     "+ str(dec)+"     "+str(radius)+"\n")
            # output.write("{}     {}     {}     {} \n".format(str(ra),str(dec),str(radius),pgc))
            output.write(str(ra)+"     "+str(dec)+"     "+str(radius)+"     "+pgc)
            #sys.exit()
            return -1 #special value reserved for not in SDSS footprint galaxies
        else :
            if (DEBUG): 
                print ( "Complete Query. These data lies within margin: ")
                print (data)
        # os.mkdir(filename)
        # os.chdir(filename)
        #if (os.path.exists(band)):
	    #os.system("rm -r "+band)
        os.mkdir(band)
        os.chdir(band)
        os.mkdir ("raw")
        os.mkdir ("projected")
        os.chdir("raw")
        if (DEBUG): print ("Retrieving data from SDSS SAS server for "+ band +"band")
        for i in data :  
            out = "frame-"+str(band)+"-"+str(i[0]).zfill(6)+"-"+str(i[1])+"-"+str(i[2]).zfill(4)
            os.system("wget http://mirror.sdss3.org/sas/dr10/boss/photoObj/frames/301/"+str(i[0])+"/"+str(i[1])+"/"+out+".fits.bz2")
            os.system("bunzip2 "+out+".fits.bz2")
        os.chdir("../")
        if (DEBUG) : print("Creating mosaic for "+band+" band.")
        outfile_r="SDSS_"+band+"_"+str(ra)+"_"+str(dec)+"r.fits"
        outfile="SDSS_"+band+"_"+str(ra)+"_"+str(dec)+".fits"
        if (len(data)==1):
    	    #With header info, len of processed result list is 1 if there is only 1 field lying in the margin, simply do mSubImage without mosaicing
    	    #This patch should not be necessary but the program is aparently not mosaicing for the case where there is only one field.
            print ("Only one field in region of interest")
            os.chdir("raw")
            montage.mSubimage(out+".fits",outfile,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
            #os.chdir("../..")
            hdulist = pyfits.open(outfile)
            shutil.move(outfile,"../..")
            os.chdir("../..")
        else:
            montage.mImgtbl("raw","images.tbl")
            montage.mHdr(str(ra)+" "+str(dec),margin,out+".hdr")
            if (DEBUG): print ("Reprojecting images")
            os.chdir("raw")
            montage.mProjExec("../images.tbl","../"+out+".hdr","../projected", "../stats.tbl") 
            os.chdir("..")
            montage.mImgtbl("projected","pimages.tbl")
            os.chdir("projected")
            montage.mAdd("../pimages.tbl","../"+out+".hdr","SDSS_"+out+".fits")
            # outfile_r="SDSS_{}_{}_{}r.fits".format(band,str(ra),str(dec))
            #outfile_r="SDSS_"+band+"_"+str(ra)+"_"+str(dec)+"r.fits"
            montage.mSubimage("SDSS_"+out+".fits",outfile_r,ra,dec,2*margin) # mSubImage takes xsize which should be twice the margin (margin measures center to edge of image)
            shutil.move(outfile_r,os.getcwd()[:-11] )#if change to :-11 then move out of u,g,r,i,z directory, may be more convenient for mJPEG
            if (DEBUG) : print ("Completed Mosaic for " + band)
            os.chdir("../..")
            hdulist = pyfits.open(outfile_r)
        hdulist[0].header['RA']=ra
        hdulist[0].header['DEC']=dec
        hdulist[0].header['RADIUS']=radius
        hdulist[0].header['PGC']=pgc
        hdulist[0].header['NED']=("http://ned.ipac.caltech.edu/cgi-bin/objsearch?objname="+ str(hdulist[0].header['PGC'])+"&extend=no&hconst=73&omegam=0.27&omegav=0.73&corr_z=1&out_csys=Equatorial&out_equinox=J2000.0&obj_sort=RA+or+Longitude&of=pre_text&zv_breaker=30000.0&list_limit=5&img_stamp=YES")
        hdulist[0].header['CLEAN']=clean
        hdulist[0].header['MARGIN']=margin
        
        #if (os.path.exists(outfile)):
            #os.system("rm "+ outfile)
        hdulist.writeto(outfile)
        if (os.path.exists(outfile_r)):
            os.system("rm "+outfile_r)
        #print("Deleting")
        os.system("rm -r "+band+"/")
        print ("Completed Mosaic")
        return outfile 
Example #46
0
def crossID(ra, dec, unit=None, dr=2., fields=None):
    """
    Perform object cross-ID in SDSS using SQL.
    
    Search for objects near position (ra, dec) within some radius using
    Tamas Budavari's SQL tool (sqlcl.py).
    
    Parameters
    ----------
    ra : float, int, str, tuple
        An object that represents a right ascension angle.
    dec : float, int, str, tuple
        An object that represents a declination angle.
    unit : `~astropy.units.UnitBase`, str
        The unit of the value specified for the angle
    dr : int, float
        Radius of region to perform object cross-ID (arcseconds).
    fields : list, optional
        SDSS PhotoObj or SpecObj quantities to return. If None, defaults
        to quantities required to find corresponding spectra and images
        of matched objects (e.g. plate, fiberID, mjd, etc.).
             
    See documentation for astropy.coordinates.angles for more information 
    about ('ra', 'dec', 'unit') parameters.
    
    Examples
    --------
    xid = sdss.crossID(ra='0h8m05.63s', dec='14d50m23.3s')
    
    for match in xid:
        print match['ra'], match['dec'], match['objid']

    Returns
    -------
    List of all objects found within search radius. Each element of list is 
    a dictionary containing information about each matched object.
    """

    if not isinstance(ra, coord.angles.RA):
        ra = coord.RA(ra, unit=unit)
    if not isinstance(ra, coord.angles.Dec):
        dec = coord.Dec(dec, unit=unit)

    if fields is None:
        fields = photoobj_defs + specobj_defs

    # Convert arcseconds to degrees
    dr /= 3600.

    Nfields = len(fields)

    q_select = 'SELECT '
    for field in fields:
        if field in photoobj_defs:
            q_select += 'p.%s,' % field
        if field in specobj_defs:
            q_select += 's.%s,' % field
    q_select = q_select.rstrip(',')
    q_select += ' '

    q_from = 'FROM PhotoObjAll AS p '
    q_join = 'JOIN SpecObjAll s ON p.objID = s.bestObjID '
    q_where = 'WHERE (p.ra between %g and %g) and (p.dec between %g and %g)' \
        % (ra.degrees-dr, ra.degrees+dr, dec.degrees-dr, dec.degrees+dr)

    q = sqlcl.query("%s%s%s%s" % (q_select, q_from, q_join, q_where))

    results = []
    cols = q.readline()
    while True:
        line = q.readline().replace('\n', '').split(',')

        if len(line) == 1:
            break

        tmp = {}
        for i, val in enumerate(line):

            field = fields[i]

            if val.isdigit():
                tmp[field] = int(val)
            else:
                try:
                    tmp[field] = float(val)
                except ValueError:
                    tmp[field] = str(val)

        results.append(tmp)

    return results
Example #47
0
def query_sdss_culster(file_loc, cat_ra, cat_dec, cat_z, cat_lambda,
                       name, num, start=0, plot=False,
                       spider_rad=None, spider_mean=False,
                       query_galaxy_only=True, r200_factor=None,
                       richness_mass_author=None):
    fails = []
    if(query_galaxy_only):
        query_table = "Galaxy"
    else:
        query_table = "PhotoObj"
    print("querying...")
    hfile = h5py.File(file_loc+"query_results.hdf5",mode='a')
    for i in range(start,num):
        #try:
        start = time.time()
        print("%d/%d"%(i+1,num))
        keys = hfile.keys()
        if("%s%d"%(name,i) in keys and "%s_prop%d"%(name,i) in keys):
            continue;

        if "%s%d"%(name,i) in keys:
            del hfile["%s%d"%(name,i)]

        if "%s_prop%d"%(name,i) in keys:
            del hfile["%s_prop%d"%(name,i)]
 
        #query columns are defined here:
        #http://skyserver.sdss.org/dr8/en/help/browser/browser.asp?n=PhotoObjAll&t=U
           
        ra = cat_ra[i]
        dec = cat_dec[i]
        z   = cat_z[i]
        richness = cat_lambda[i]

        # Xray Spiders have their own r200c, so we don't need to compute it. 
        if spider_rad is None: 
            mass, r200 = background.lambda_to_m200_r200(richness,z, richness_mass_author=richness_mass_author)
            rad = background.r200_to_arcmin(r200, z)
        else:
            r200c_deg = spider_rad[i]
            rad = r200c_deg * 60
            r200 = background.arcmin_to_r200(rad, z)
            mass = background.r200c_to_m200c(r200, z)
            if spider_mean:
                m200m, r200m, c200m =mass_adv.changeMassDefinitionCModel(mass, z, 
                                                                         "200c", "200m",
                                                                         c_model='child18')
                mass = m200m
                r200m_r200c_ratio = r200m/r200
                rad *= r200m_r200c_ratio
                r200 *= r200m_r200c_ratio
                

        hgroup = hfile.create_group("%s_prop%d"%(name,i))
        hgroup.create_dataset("ra",data=ra)
        hgroup.create_dataset("dec",data=dec)
        hgroup.create_dataset("z",data=z)
        hgroup.create_dataset("mass",data=mass)
        hgroup.create_dataset("rad",data=rad)
        hgroup.create_dataset("r200",data=r200)
        hgroup.create_dataset("richness", data = richness)
        ## Query and save objects around target

        query_str = "select  p.ra, p.dec, p.type,p.insidemask,p.flags_g,p.flags_i,p.flags_r,p.cModelMag_u, p.cModelMagErr_u,p.cModelMag_g, p.cModelMagErr_g,p.cModelMag_r, p.cModelMagErr_r,p.cModelMag_i, p.cModelMagErr_i,p.cModelMag_z, p.cModelMagErr_z from "+query_table+" p join dbo.fGetNearbyObjEq(%f,%f,%f) r on p.ObjID = r.ObjID"%(ra, dec, r200_factor*rad)

        result = sqlcl.query(query_str).read()
        # datagal = np.genfromtxt(StringIO(result),names=True,delimiter=', ',dtype=['f8','f8','i2','i1','i8','i8','i8', 'f4','f4', 'f4','f4', 'f4','f4', 'f4','f4', 'f4','f4'])
        try:
            datagal = np.genfromtxt(StringIO(result),names=True,skip_header=1,delimiter=',',dtype=['f8','f8','i2','i1','i8','i8','i8', 'f4','f4', 'f4','f4', 'f4','f4', 'f4','f4', 'f4','f4'])
        except ValueError as e:
            print(query_str)
            continue

        hgroup = hfile.create_group("%s%d"%(name,i))
        hgroup.create_dataset("ra",data=datagal['ra'])
        hgroup.create_dataset("dec",data=datagal['dec'])
        hgroup.create_dataset("type",data=datagal['type'])
        hgroup.create_dataset("insidemask",data=datagal['insidemask'])
        hgroup.create_dataset("flags_g",data=datagal['flags_g'])
        hgroup.create_dataset("flags_i",data=datagal['flags_i'])
        hgroup.create_dataset("flags_r",data=datagal['flags_r'])
        hgroup.create_dataset("mag_u",data=datagal['cModelMag_u'])
        hgroup.create_dataset("mag_err_u",data=datagal['cModelMagErr_u'])
        hgroup.create_dataset("mag_g",data=datagal['cModelMag_g'])
        hgroup.create_dataset("mag_err_g",data=datagal['cModelMagErr_g'])
        hgroup.create_dataset("mag_r",data=datagal['cModelMag_r'])
        hgroup.create_dataset("mag_err_r",data=datagal['cModelMagErr_r'])
        hgroup.create_dataset("mag_i",data=datagal['cModelMag_i'])
        hgroup.create_dataset("mag_err_i",data=datagal['cModelMagErr_i'])
        hgroup.create_dataset("mag_z",data=datagal['cModelMag_z'])
        hgroup.create_dataset("mag_err_z",data=datagal['cModelMagErr_z'])

        end = time.time()
        print(" time: %.2f"%float(end-start))
        if(plot):
            plt.figure()
            legends = ["uknown","cosmic_ray","defect","galaxy","ghost","knownobj","star","trail","sky","notatype"]
            slct1 = datagal['insidemask']==0
            for i in range(0,10):
                slct = (datagal["type"] == i) & slct1
                plt.plot(datagal['ra'][slct],datagal['dec'][slct],'x',label=legends[i])
            plt.legend(loc='best')
            plt.xlabel('ra')
            plt.ylabel('dec')
            plt.show()
        # except ValueError as ie:
        #     print ie
        #     fails.append(i)
        #     print "Failure"
        np.save(file_loc+"fail_indexs.npy",fails)
    hfile.close();
Example #48
0
def sdss_sql():

    #i = -1 
    for q in range(Nz):
        '''
        i = i+1
        if ( i % 10 != 0 ):
            continue
        '''
        time = z[q]
        ra = Ra[q]
        dec = Dec[q]

        c_ra0 = str(ra - r_select)
        c_dec0 = str(dec - r_select)
        c_ra1 = str(ra + r_select)
        c_dec1 = str(dec + r_select)

        qry = """
        SELECT ALL
        p.ra,p.dec,p.u,p.g,p.r,p.i,p.z,p.type,  
        p.probPSF,
        p.petroR90_u, p.petroR90_g, p.petroR90_r, p.petroR90_i, p.petroR90_z,

        p.deVRad_u, p.deVRad_g, p.deVRad_r, p.deVRad_i, p.deVRad_z,
        p.deVAB_u, p.deVAB_g, p.deVAB_r, p.deVAB_i, p.deVAB_z,
        p.deVPhi_u, p.deVPhi_g, p.deVPhi_r, p.deVPhi_i, p.deVPhi_z,

        p.expRad_u, p.expRad_g, p.expRad_r, p.expRad_i, p.expRad_z,
        p.expAB_u, p.expAB_g, p.expAB_r, p.expAB_i, p.expAB_z,
        p.expPhi_u, p.expPhi_g, p.expPhi_r, p.expPhi_i, p.expPhi_z 
        FROM PhotoObj AS p
        WHERE
           p.ra BETWEEN %s AND %s
           AND p.dec BETWEEN %s AND %s
           AND p.type = 6
        ORDER by p.r
        """ % (c_ra0, c_ra1, c_dec0, c_dec1)
        '''
        # add date: 2019.8.30
        qry = """
        SELECT ALL
            p.ra, p.dec, p.u, p.g, p.r, p.i, p.z, p.type,
            p.isoA_u, p.isoA_g, p.isoA_r, p.isoA_i, p.isoA_z,
            p.isoB_u, p.isoB_g, p.isoB_r, p.isoB_i, p.isoB_z,
            p.isoPhi_u, p.isoPhi_g, p.isoPhi_r, p.isoPhi_i, p.isoPhi_z,
            p.flags, dbo.fPhotoFlagsN(p.flags)
        FROM PhotoObj AS p
        WHERE
            p.ra BETWEEN %s AND %s AND p.dec BETWEEN %s AND %s
            AND (p.type = 6 OR (p.flags & dbo.fPhotoFlags('SATURATED')) > 0)
        ORDER by p.r
        """ % (c_ra0, c_ra1, c_dec0, c_dec1)
        '''
        cord_z = z_cod[q]
        cord_ra = ra_cod[q]
        cord_dec = dec_cod[q]
        print( qry )
        file = sqlcl.query( qry, url, fmt )
 
        #fd = open( '/home/xkchen/mywork/ICL/data/star_catalog/ra%.3f_dec%.3f_z%.3f.csv'%(cord_ra, cord_dec, cord_z), 'w' )
        fd = open( './ra%.3f_dec%.3f_z%.3f.csv'%(cord_ra, cord_dec, cord_z), 'w' )

        #sqlcl.write_header( fd, "#", url, qry )

        lines = file.readlines()

        for l in lines:
            dtl = l.decode('utf-8')
            fd.write( dtl )
        fd.close()

        if q == 5:
            break

    return
def search_sdss(blazardir='/Users/willettk/Astronomy/Research/blazars/',blazarfile='plotkin_dr8_upload.txt',savstring='',envsize=1000,ngals=1000, save=False, timing=True):
	
	import sqlcl
	import cosmocalc
	import time

	timestart = time.time()
	
	# Read in the list of blazars from combined catalogs (Plotkin, BZCAT, TeVcat)
	
	pf = open(blazardir+blazarfile,'r')
	
	# Python dictionary for the relevant data to be returned
	
	sdss_data_zns = { 'objid':[], 'ra':[], 'dec':[], 'type':[], 'nchild':[],
		'u':[], 'g':[], 'r':[], 'i':[], 'z':[],
		'p_el':[], 'p_cw':[], 'p_acw':[], 'p_edge':[], 'p_dk':[], 'p_mg':[], 'p_cs':[],
		'blazar_name':[]
		}
	
	sdss_data_zs = { 'objid':[], 'ra':[], 'dec':[], 'type':[], 'nchild':[],
		'u':[], 'g':[], 'r':[], 'i':[], 'z':[],
		'p_el':[], 'p_cw':[], 'p_acw':[], 'p_edge':[], 'p_dk':[], 'p_mg':[], 'p_cs':[],
		'blazar_name':[]
		}
	
	sdss_data_nozoo = { 'objid':[], 'ra':[], 'dec':[], 'type':[], 'nchild':[],
		'u':[], 'g':[], 'r':[], 'i':[], 'z':[],
		'blazar_name':[]
		}
	
	errorarr = {'type':[],'name':[]}
	
	for line in pf:
		#
		blazar_name, blazar_ra, blazar_dec, blazar_z, blazar_type = line.split()
		blazar_z = float(blazar_z)
		#
		# Compute the angular size of 1000 kpc
		#
		platescale = cosmocalc.cosmocalc(z=blazar_z, H0=71, WM=0.27)['PS_kpc'] # kpc per arcsec
		angsize = envsize / platescale / 60.
		#
		# Query the SDSS DR8 catalog
		#
		blazar_query_zns = """ SELECT top %i p.objID, p.ra, p.dec, p.type, p.nchild, p.u, p.g, p.r, p.i, p.z, zns.p_el, zns.p_cw, zns.p_acw, zns.p_edge, zns.p_dk, zns.p_mg, zns.p_cs FROM fGetNearbyObjEq(%f,%f,%f) n JOIN PhotoObj p on p.objid = n.objid JOIN zooNoSpec zns on zns.objid = n.objid ORDER BY p.objID """ % (ngals, float(blazar_ra), float(blazar_dec), angsize)
		blazar_query_zs = """ SELECT top %i p.objID, p.ra, p.dec, p.type, p.nchild, p.u, p.g, p.r, p.i, p.z, zs.p_el, zs.p_cw, zs.p_acw, zs.p_edge, zs.p_dk, zs.p_mg, zs.p_cs FROM fGetNearbyObjEq(%f,%f,%f) n JOIN PhotoObj p on p.objid = n.objid JOIN zooSpec zs on zs.objid = n.objid ORDER BY p.objID """ % (ngals, float(blazar_ra), float(blazar_dec), angsize)
		blazar_query_nozoo = """ SELECT top %i p.objID, p.ra, p.dec, p.type, p.nchild, p.u, p.g, p.r, p.i, p.z FROM fGetNearbyObjEq(%f,%f,%f) n JOIN PhotoObj p on p.objid = n.objid ORDER BY p.objID """ % (ngals, float(blazar_ra), float(blazar_dec), angsize)
		#
                bqtime = time.time()
		#
		queryreturn_zns = sqlcl.query(blazar_query_zns, fmt='csv')
		queryreturn_zs = sqlcl.query(blazar_query_zs, fmt='csv')
		queryreturn_nozoo = sqlcl.query(blazar_query_nozoo, fmt='csv')
		#
		lqtime = time.time()
		#
		# Make sure queries do not exceed 60 per minute (intrinsic limit of database)
		#
		if (lqtime - bqtime) < 3.1:
		        time.sleep(3.1 - (lqtime - bqtime))
		#
		qr_read_zns = queryreturn_zns.read()
		qr_read_zs = queryreturn_zs.read()
		qr_read_nozoo = queryreturn_nozoo.read()
		#
		# Print detections to the screen
		#
		if (qr_read_zns[:26] != 'No objects have been found') and (qr_read_zns[:5] != 'ERROR'):
			print blazar_name
			#
			qrsplit = qr_read_zns.split()
			print qrsplit[1:]
			for n in arange(len(qrsplit[1:]))+1:
				neighbor_data = qrsplit[n].split(',')
				sdss_data_zns['objid'].append(neighbor_data[0])
				sdss_data_zns['ra'].append(neighbor_data[1])
				sdss_data_zns['dec'].append(neighbor_data[2])
				sdss_data_zns['type'].append(neighbor_data[3])
				sdss_data_zns['nchild'].append(neighbor_data[4])
				sdss_data_zns['u'].append(neighbor_data[5])
				sdss_data_zns['g'].append(neighbor_data[6])
				sdss_data_zns['r'].append(neighbor_data[7])
				sdss_data_zns['i'].append(neighbor_data[8])
				sdss_data_zns['z'].append(neighbor_data[9])
				sdss_data_zns['p_el'].append(neighbor_data[10])
				sdss_data_zns['p_cw'].append(neighbor_data[11])
				sdss_data_zns['p_acw'].append(neighbor_data[12])
				sdss_data_zns['p_edge'].append(neighbor_data[13])
				sdss_data_zns['p_dk'].append(neighbor_data[14])
				sdss_data_zns['p_mg'].append(neighbor_data[15])
				sdss_data_zns['p_cs'].append(neighbor_data[16])
				sdss_data_zns['blazar_name'].append(blazar_name)
	
	
		if (qr_read_zs[:26] != 'No objects have been found') and (qr_read_zs[:5] != 'ERROR'):
			print blazar_name
			#
			qrsplit = qr_read_zs.split()
			print qrsplit[1:]
			for n in arange(len(qrsplit[1:]))+1:
				neighbor_data = qrsplit[n].split(',')
				sdss_data_zs['objid'].append(neighbor_data[0])
				sdss_data_zs['ra'].append(neighbor_data[1])
				sdss_data_zs['dec'].append(neighbor_data[2])
				sdss_data_zs['type'].append(neighbor_data[3])
				sdss_data_zs['nchild'].append(neighbor_data[4])
				sdss_data_zs['u'].append(neighbor_data[5])
				sdss_data_zs['g'].append(neighbor_data[6])
				sdss_data_zs['r'].append(neighbor_data[7])
				sdss_data_zs['i'].append(neighbor_data[8])
				sdss_data_zs['z'].append(neighbor_data[9])
				sdss_data_zs['p_el'].append(neighbor_data[10])
				sdss_data_zs['p_cw'].append(neighbor_data[11])
				sdss_data_zs['p_acw'].append(neighbor_data[12])
				sdss_data_zs['p_edge'].append(neighbor_data[13])
				sdss_data_zs['p_dk'].append(neighbor_data[14])
				sdss_data_zs['p_mg'].append(neighbor_data[15])
				sdss_data_zs['p_cs'].append(neighbor_data[16])
				sdss_data_zs['blazar_name'].append(blazar_name)
	
	
		if (qr_read_nozoo[:26] != 'No objects have been found') and (qr_read_nozoo[:5] != 'ERROR'):
			print blazar_name
			#
			qrsplit = qr_read_nozoo.split()
			print qrsplit[1:]
			for n in arange(len(qrsplit[1:]))+1:
				neighbor_data = qrsplit[n].split(',')
				sdss_data_nozoo['objid'].append(neighbor_data[0])
				sdss_data_nozoo['ra'].append(neighbor_data[1])
				sdss_data_nozoo['dec'].append(neighbor_data[2])
				sdss_data_nozoo['type'].append(neighbor_data[3])
				sdss_data_nozoo['nchild'].append(neighbor_data[4])
				sdss_data_nozoo['u'].append(neighbor_data[5])
				sdss_data_nozoo['g'].append(neighbor_data[6])
				sdss_data_nozoo['r'].append(neighbor_data[7])
				sdss_data_nozoo['i'].append(neighbor_data[8])
				sdss_data_nozoo['z'].append(neighbor_data[9])
				sdss_data_nozoo['blazar_name'].append(blazar_name)
	
	
		if qr_read_zns[:5] == 'ERROR':
			errorarr['type'].append('zns')
			errorarr['name'].append(blazar_name)
	
		if qr_read_zs[:5] == 'ERROR':
			errorarr['type'].append('zs')
			errorarr['name'].append(blazar_name)
	
		if qr_read_nozoo[:5] == 'ERROR':
			errorarr['type'].append('nozoo')
			errorarr['name'].append(blazar_name)

	if save is True:

		import pickle

		# Save dictionary to file so it doesn't have to be rerun
		
		output = open('sdss_'+str(envsize)+'kpc_zns'+str(savstring)+'.pkl', 'w')
		pickle.dump(sdss_data_zns, output)
		output.close()
		
		output = open('sdss_'+str(envsize)+'kpc_zs'+str(savstring)+'.pkl', 'w')
		pickle.dump(sdss_data_zs, output)
		output.close()
		
		output = open('sdss_'+str(envsize)+'kpc_nozoo'+str(savstring)+'.pkl', 'w')
		pickle.dump(sdss_data_nozoo, output)
		output.close()

		print "Saved dictionaries to file"
	
	endtime = time.time()

	print 'Total time elapsed is %5.2f min' % ((endtime - starttime)*60)
def query_sdss_mask(file_loc,
                    cat_ra,
                    cat_dec,
                    cat_z,
                    cat_lambda,
                    name,
                    num,
                    r200_factor=1.0,
                    start=0,
                    plot=False,
                    save_data=True,
                    spider_rad=None,
                    spider_mean=False,
                    richness_mass_author='Rykoff'):
    global num_pass
    global num_fail
    fails = []
    print file_loc + name + "_mask.hdf5"
    if (save_data):
        hfile = h5py.File(file_loc + name + "_mask.hdf5", 'w')

    for i in range(start, num):
        start = time.time()
        #query columns are defined here:
        #http://skyserver.sdss.org/dr8/en/help/browser/browser.asp?n=PhotoObjAll&t=U
        print "%d/%d " % (i, num),

        ra = cat_ra[i]
        dec = cat_dec[i]
        z = cat_z[i]
        richness = cat_lambda[i]
        mass, r200 = background.lambda_to_m200_r200(
            richness, z, richness_mass_author=richness_mass_author)
        if spider_rad is None:
            r200_deg = background.r200_to_arcmin(r200, z) / 60.0
            rad = background.r200_to_arcmin(r200, z) * r200_factor
        else:
            r200c_deg = spider_rad[i]
            rad = r200c_deg * 60
            r200 = background.arcmin_to_r200(rad, z)
            mass = background.r200c_to_m200c(r200, z)
            if spider_mean:
                m200m, r200m, c200m = mass_adv.changeMassDefinitionCModel(
                    mass, z, "200c", "200m", c_model='child18')
                mass = m200m
                r200m_r200c_ratio = r200m / r200
                rad *= r200m_r200c_ratio
                r200 *= r200m_r200c_ratio

        print "ra", ra, "dec", dec
        print "z", z, "l", richness, "mass", mass, "r200", r200,
        ## Query and save objects around target
        rad_deg = rad / 60.0
        rad_mask = rad_deg * 3
        a = np.cos(np.pi * dec / 180.0)
        query_str = "SELECT ra, dec, radius, type,area FROM Mask where ra < %f and ra > %f and dec < %f and dec > %f and type != 4" % (
            ra + rad_mask / a, ra - rad_mask / a, dec + rad_mask,
            dec - rad_mask)
        result = sqlcl.query(query_str)
        result.readline()
        try:
            data_mask = np.genfromtxt(StringIO(result.read()),
                                      names=True,
                                      delimiter=",",
                                      dtype=['f4', 'f4', 'f4', 'i4', 'S500'])
            mask_pass = mask_outside_r200(ra, dec, rad_deg, data_mask['area'],
                                          data_mask['type'])
        except ValueError as e:
            print("failed query, auto fail")
            mask_pass = False
        if (mask_pass):
            num_pass += 1
        else:
            num_fail += 1
        print "\tpass:"******"\tfract total: %.3f " % (
            float(num_pass) / float(num_pass + num_fail)),
        if (save_data):
            hgroup = hfile.create_group(str(i))
            dataset = hgroup.require_dataset("mask_pass", (1, 1), 'u1',
                                             mask_pass)
            if (mask_pass):
                dataset[0, 0] = True
            else:
                dataset[0, 0] = False
            dt_str = h5py.special_dtype(vlen=str)
            dataset = hgroup.require_dataset("mask_points",
                                             (data_mask["area"].size, ),
                                             dtype=dt_str)
            if (data_mask["area"].size == 1):
                dataset[0] = data_mask['area']
            else:
                for j in range(0, data_mask["area"].size):
                    dataset[j] = data_mask["area"][j]

        if (plot):
            plt.figure()
            plt.title("pass: "******"cirlce test: dist: ",dist,"ra,dec: ",ras[j],decs[j]
            # plt.plot(ras,decs,'-k')
            for j in range(0, data_mask['area'].size):
                if (data_mask['area'].size == 1):
                    x, y = area_str_to_lines(str(data_mask['area']))
                    mask_type = data_mask['type']
                else:
                    x, y = area_str_to_lines(data_mask['area'][j])
                    mask_type = data_mask['type'][j]
                    if (mask_type == 0):
                        plot_type = 'r'
                    elif (mask_type == 1):
                        plot_type = 'b'
                    elif (mask_type == 2):
                        plot_type = 'g--'
                    elif (mask_type == 3):
                        plot_type = 'k:'
                    plt.plot(x, y, plot_type)
            plt.plot([], [], 'r', label='bleeding')
            plt.plot([], [], 'b', label='bright star')
            plt.plot([], [], 'g--', label='trail')
            plt.plot([], [], 'k:', label='quality hole')

            plt.legend(loc='best')
            plt.xlabel('ra')
            plt.ylabel('dec')
            plt.xlim([ra - rad_mask, ra + rad_mask])
            plt.ylim([dec - rad_mask, dec + rad_mask])
            plt.show()
        #close the file we have been writing to.
        end = time.time()
        print " time: %.2f" % float(end - start)
    if (save_data):
        hfile.close()
Example #51
0
def ObsRealism(
    inputName,
    outputName,
    band='r',
    cosmo=FlatLambdaCDM(H0=70, Om0=0.3),
    common_args={
        'redshift': 0.1,  # mock observation redshift
        'rebin_to_CCD': False,  # rebin to CCD angular scale
        'CCD_scale': 0.396,  # CCD angular scale in [arcsec/pixel]
        'add_false_sky': False,  # add gaussian sky
        'false_sky_sig': 24.2,  # gaussian sky standard dev [AB mag/arcsec2]
        'add_false_psf': False,  # convolve with gaussian psf
        'false_psf_fwhm': 1.0,  # gaussian psf FWHM [arcsec]
        'add_poisson': False,  # add poisson noise to galaxy
        'add_sdss_sky': False,  # insert into real SDSS sky (using sdss_args)
        'add_sdss_psf': False,  # convolve with real SDSS psf (using sdss_args)
    },
    sdss_args={
        'sdss_run': 745,  # sdss run
        'sdss_rerun': 40,  # sdss rerun
        'sdss_camcol': 1,  # sdss camcol
        'sdss_field': 517,  # sdss field
        'sdss_ra': 236.1900,  # ra for image centroid
        'sdss_dec': -0.9200,  # ec for image centroid
    }):
    '''
    Add realism to idealized unscaled image.
    
    "redshift": The redshift at which the synthetic image is to be mock-observed. Given that the image should be in surface brightness units and appropriately dimmed by (1+z)^-5, the redshift is only used to determine the angular-to-physical scale of the image -- to which it is appropriately rebinned corresponding to the desired CCD pixel scale.
    
    "rebin_to_CCD": If TRUE, the image is rebinned to the CCD scale identified by the "CCD_scale" keyword. The rebinning is determined by first computing the physical-to-angular scale associated with the target redshift [kpc/arcsec]. Combining this number with the scale of the original image in physical units [kpc/pixel], we obtain the rebinning factor that is neccesary to bring the image to the desired CCD pixel scale [arcsec/pixel].
    
    "CCD_scale": The CCD scale to which the images are rebinned if rebin_to_CCD is TRUE.
    
    "add_false_sky": If TRUE, a Gaussian sky is added to the image with a noise level that is idenfitied by the "false_sky_sig" keyword.
    
    "false_sky_sig": The standard deviation of Gaussian sky that is added to the image if "add_false_sky" is TRUE. The value must be expressed in relative magnitude units (AB mag/arcsec2).
    
    "add_false_psf": If TRUE, a Gaussian PSF is added to the image with a FWHM that is idenfitied by the "false_psf_fwhm" keyword.
    
    "false_psf_fwhm": The FWHM of the PSF that is convolved with the image if "add_false_psf" is TRUE. The value must be expressed in arcsec.
    
    "add_poisson": If TRUE, add Poisson noise to the image using either the calibration info and gain from the real image properties ("add_sdss_sky"=TRUE) or generic values derived from averages over SDSS fields.
    
    "add_sdss_sky": If True, insert into real SDSS sky using arguments in "sdss_args".
    
    "add_sdss_psf": If True and "add_sdss_sky"=True, reconstruct the PSF at the injection location and convolve with the image.
    '''

    # mock observation redshift
    redshift = common_args['redshift']
    # speed of light [m/s]
    speed_of_light = 2.99792458e8
    # kiloparsec per arcsecond scale
    kpc_per_arcsec = cosmo.kpc_proper_per_arcmin(
        z=redshift).value / 60.  # [kpc/arcsec]
    # luminosity distance in Mpc
    luminosity_distance = cosmo.luminosity_distance(z=redshift)  # [Mpc]

    # img header and data
    with fits.open(inputName, mode='readonly') as hdul:
        # img header
        header = hdul[0].header
        # img data
        img_data = hdul[0].data

#    # header properties
#    sim_tag = header['SIMTAG']
#    sub_tag = header['SUBTAG']
#    isnap = header['ISNAP']
#    axis = header['CAMERA']
#    band = header['FILTER'][0]
#
#    # unique simulID
#    simulID = '{}-{}-{}-{}'.format(sim_tag,sub_tag,isnap,axis)
#
#    band = header['FILTER'][0]

# collect physical pixel scale
    kpc_per_pixel = header['CDELT1'] / 1000.  # [kpc/pixel]
    # compute angular pixel scale from cosmology
    arcsec_per_pixel = kpc_per_pixel / kpc_per_arcsec  # [arcsec/pixel]

    # img in AB nanomaggies per arcsec2
    img_nanomaggies = 10**(-0.4 * (img_data - 22.5))  # [nmgys/arcsec2]
    # apply pixel scale [arcsec/pixel]2 to convert to calibrated flux
    img_nanomaggies *= arcsec_per_pixel**2  # [nmgs]
    # update units of image header to linear calibrated scale
    header['BUNIT'] = 'AB nanomaggies'

    #    print('\nRaw image:')
    #    print('kpc_per_arcsec: {}'.format(kpc_per_arcsec))
    #    print('kpc_per_pixel: {}'.format(kpc_per_pixel))
    #    print('arcsec_per_pixel: {}'.format(arcsec_per_pixel))
    #    m_AB = -2.5*np.log10(np.sum(img_nanomaggies))+22.5
    #    print('AB_magnitude: {} at z={}'.format(m_AB,redshift))
    #    M_AB = m_AB-5*np.log10(luminosity_distance.value)-25
    #    print('AB_Magnitude: {}'.format(M_AB))

    # Add levels of realism

    if common_args['rebin_to_CCD']:
        '''
        Rebin image to a given angular CCD scale
        '''
        # telescope ccd angular scale
        ccd_scale = common_args['CCD_scale']
        # axes of original image
        nPixelsOld = img_nanomaggies.shape[0]
        # axes of regridded image
        nPixelsNew = int(np.floor((arcsec_per_pixel / ccd_scale) * nPixelsOld))
        # rebin to new ccd scale
        if nPixelsNew > nPixelsOld:
            interp = RectBivariateSpline(np.linspace(-1, 1, nPixelsOld),
                                         np.linspace(-1, 1, nPixelsOld),
                                         img_nanomaggies,
                                         kx=1,
                                         ky=1)
            img_nanomaggies = interp(np.linspace(
                -1, 1, nPixelsNew), np.linspace(
                    -1, 1, nPixelsNew)) * (nPixelsOld / nPixelsNew)**2
        else:
            img_nanomaggies = rebin(img_nanomaggies, (nPixelsNew, nPixelsNew))
        # new kpc_per_pixel on ccd
        kpc_per_pixel = kpc_per_arcsec * ccd_scale
        # new arcsec per pixel
        arcsec_per_pixel = ccd_scale
        # header updates
        if nPixelsNew % 2: CRPIX = float(nPixelsNew / 2)
        else: CRPIX = float(nPixelsNew / 2) + 0.5
        header['CRPIX1'] = CRPIX
        header['CRPIX2'] = CRPIX
        header['CDELT1'] = kpc_per_pixel * 1000
        header['CDELT2'] = kpc_per_pixel * 1000


#        print('\nAfter CCD scaling:')
#        print('kpc_per_arcsec: {}'.format(kpc_per_arcsec))
#        print('kpc_per_pixel: {}'.format(kpc_per_pixel))
#        print('arcsec_per_pixel: {}'.format(arcsec_per_pixel))
#        m_AB = -2.5*np.log10(np.sum(img_nanomaggies))+22.5
#        print('AB_magnitude: {} at z={}'.format(m_AB,redshift))
#        M_AB = m_AB-5*np.log10(luminosity_distance.value)-25
#        print('AB_Magnitude: {}'.format(M_AB))

# convolve with gaussian psf
    if common_args['add_false_psf']:
        '''
        Add Gaussian PSF to image with provided FWHM in
        arcseconds.
        '''
        std = common_args['false_psf_fwhm'] / arcsec_per_pixel / 2.355
        kernel = Gaussian2DKernel(stddev=std)
        img_nanomaggies = convolve(img_nanomaggies, kernel)

    # add poisson noise to image
    if common_args['add_poisson'] and not common_args['add_sdss_sky']:
        '''
        Add shot noise to image assuming the average SDSS
        field properties for zeropoint, airmass, atmospheric
        extinction, and gain. The noise calculation assumes
        that the number of counts in the converted image is 
        the mean number of counts in the Poisson distribution.
        Thereby, the standard error in that number of counts 
        is the square root of the number of counts in each 
        pixel.
        
        For details on the methods applied here, see:
        http://classic.sdss.org/dr7/algorithms/fluxcal.html
        
        Average quantites obtained from SkyServer SQL form.
        http://skyserver.sdss.org/dr7/en/tools/search/sql.asp
        DR7 Query Form:
        SELECT AVG(airmass_x),AVG(aa_x),AVG(kk_x),AVG(gain_x)
        FROM Field
        '''
        # average sdss photometric field properties (gain is inverse gain)
        airmass = {'u': 1.178, 'g': 1.178, 'r': 1.177, 'i': 1.177, 'z': 1.178}
        aa = {'u': -23.80, 'g': -24.44, 'r': -24.03, 'i': -23.67, 'z': -21.98}
        kk = {'u': 0.5082, 'g': 0.1898, 'r': 0.1032, 'i': 0.0612, 'z': 0.0587}
        gain = {'u': 1.680, 'g': 3.850, 'r': 4.735, 'i': 5.111, 'z': 4.622}
        exptime = 53.907456  # seconds
        # conversion factor from nanomaggies to counts
        counts_per_nanomaggy = exptime * 10**(
            -0.4 * (22.5 + aa[band] + kk[band] * airmass[band]))
        # image in counts for given field properties
        img_counts = np.clip(img_nanomaggies * counts_per_nanomaggy,
                             a_min=0,
                             a_max=None)
        # poisson noise [adu] computed accounting for gain [e/adu]
        img_counts = np.random.poisson(lam=img_counts *
                                       gain[band]) / gain[band]
        # convert back to nanomaggies
        img_nanomaggies = img_counts / counts_per_nanomaggy

    # add gaussian sky to image
    if common_args['add_false_sky']:
        '''
        Add sky with noise level set by "false_sky_sig" 
        keyword. "false_sky_sig" should be in relative  
        AB magnitudes/arcsec2 units. In other words,
        10**(-0.4*false_sky_sig) gives the sample 
        standard deviation in the sky in linear flux units
        [maggies/arcsec2] around a sky level of zero.
        '''
        # sky sig in AB mag/arcsec2
        false_sky_sig = common_args['false_sky_sig']
        # conversion from mag/arcsec2 to nanomaggies/arcsec2
        false_sky_sig = 10**(0.4 * (22.5 - false_sky_sig))
        # account for pixel scale in final image
        false_sky_sig *= arcsec_per_pixel**2
        # create false sky image
        sky = false_sky_sig * np.random.randn(*img_nanomaggies.shape)
        # add false sky to image in nanomaggies
        img_nanomaggies += sky

    # add image to real sdss sky
    if common_args['add_sdss_sky']:
        '''
        Extract field from galaxy survey database using
        effectively weighted by the number of galaxies in
        each field. For this to work, the desired field
        mask should already have been generated and the
        insertion location selected.
        '''
        import sqlcl
        from astropy.wcs import WCS
        run = sdss_args['sdss_run']
        rerun = sdss_args['sdss_rerun']
        camcol = sdss_args['sdss_camcol']
        field = sdss_args['sdss_field']
        ra = sdss_args['sdss_ra']
        dec = sdss_args['sdss_dec']
        exptime = 53.907456  # seconds

        # sdss data archive server
        das_url = 'http://das.sdss.org/'

        # get and uzip corrected image
        corr_url = das_url + 'imaging/{}/{}/corr/{}/'.format(
            run, rerun, camcol)
        corr_image_name = 'fpC-{:06}-{}{}-{:04}.fit'.format(
            run, band, camcol, field)
        if not os.access(corr_image_name, 0):
            corr_url += '{}.gz'.format(corr_image_name)
            os.system('wget {}'.format(corr_url))
            os.system('gunzip {}'.format(corr_image_name))
        # get wcs mapping
        w = WCS(corr_image_name)
        # determine column and row position in image
        colc, rowc = w.all_world2pix(ra, dec, 1, ra_dec_order=True)
        # convert to integers
        colc, rowc = int(np.around(colc)), int(np.around(rowc))

        # get field properties from skyServer
        dbcmd = [
            'SELECT aa_{b},kk_{b},airmass_{b},gain_{b},sky_{b},skysig_{b}'.
            format(b=band),
            'FROM Field where run={} AND rerun={}'.format(run, rerun),
            'AND camcol={} AND field={}'.format(camcol, field)
        ]
        lines = sqlcl.query(' '.join(dbcmd)).readlines()
        # zeropoint, atmospheric extinction, airmass, inverse gain, sky, sky uncertainty
        aa, kk, airmass, gain, sky, skysig = [
            float(var)
            for var in lines[1].decode("utf-8").split('\n')[0].split(',')
        ]
        #print(aa,kk,airmass,gain,sky,skysig)
        # convert sky to nanomaggies from maggies/arcsec2
        sky *= (1e9 * 0.396127**2)
        # convert skysig to nanomaggies from relative sky magnitude errors
        skysig *= sky * np.log(10) / 2.5
        # software bias added to corrected images to avoid negative values
        softbias = float(fits.getheader(corr_image_name)['SOFTBIAS'])
        # subtract softbias from corrected image to get image in DN
        corr_image_data = fits.getdata(corr_image_name).astype(
            float) - softbias  # [counts]
        # conversion from nanomaggies to counts
        counts_per_nanomaggy = exptime * 10**(-0.4 *
                                              (22.5 + aa + kk * airmass))
        # convert image in counts to nanomaggies with Field properties
        corr_image_data /= counts_per_nanomaggy  # [nanomaggies]

        if common_args['add_sdss_psf'] and not common_args['add_false_psf']:
            '''
            Grab, reconstruct, and convolve real SDSS PSF image
            with the image in nanomaggies.
            '''
            # get corresponding psf reconstruction image
            psf_url = das_url + 'imaging/{}/{}/objcs/{}/'.format(
                run, rerun, camcol)
            psf_image_name = 'psField-{:06}-{}-{:04}.fit'.format(
                run, camcol, field)
            if os.access(psf_image_name, 0): os.remove(psf_image_name)
            psf_url += psf_image_name
            os.system('wget {}'.format(psf_url))
            psf_ext = {'u': 1, 'g': 2, 'r': 3, 'i': 4, 'z': 5}
            psfname = 'sdss_psf.fit'
            os.system(
                '{}/Sources/utils/sdss-apps/readAtlasImages-v5_4_11/read_PSF {} {} {} {} {}'
                .format(realsim_dir, psf_image_name, psf_ext[band], rowc, colc,
                        psfname))
            if os.access(psf_image_name, 0): os.remove(psf_image_name)
            # remove softbias from PSF
            psfdata = fits.getdata(psfname).astype(float) - 1000.
            # normalize for convolution with image in nanomaggies
            psfdata /= np.sum(psfdata)
            # convolve with image in nanomaggies
            img_nanomaggies = convolve(img_nanomaggies, psfdata)
            if os.access(psfname, 0): os.remove(psfname)

        if common_args['add_poisson']:
            '''
            Add Poisson noise to the PSF-convolved image
            with noise level corresponding to the real SDSS
            field properties.
            '''
            # image in counts for given field properties
            img_counts = np.clip(img_nanomaggies * counts_per_nanomaggy,
                                 a_min=0,
                                 a_max=None)
            # poisson noise [adu] computed accounting for gain [e/adu]
            img_counts = np.random.poisson(lam=img_counts * gain) / gain
            # convert back to nanomaggies
            img_nanomaggies = img_counts / counts_per_nanomaggy

        # add real sky pixel by pixel to image in nanomaggies
        corr_ny, corr_nx = corr_image_data.shape
        ny, nx = img_nanomaggies.shape
        for xx in range(nx):
            for yy in range(ny):
                corr_x = int(colc - nx / 2 + xx)
                corr_y = int(rowc - ny / 2 + yy)
                if corr_x >= 0 and corr_x <= corr_nx - 1 and corr_y >= 0 and corr_y <= corr_ny - 1:
                    img_nanomaggies[yy, xx] += corr_image_data[corr_y, corr_x]
                else:
                    img_nanomaggies[yy, xx] == 0.
        if os.access(corr_image_name, 0): os.remove(corr_image_name)

        # add field info to image header
        warnings.simplefilter('ignore', category=AstropyWarning)
        header.append(('RUN', run, 'SDSS image RUN'), end=True)
        header.append(('RERUN', rerun, 'SDSS image RERUN'), end=True)
        header.append(('CAMCOL', camcol, 'SDSS image CAMCOL'), end=True)
        header.append(('FIELD', field, 'SDSS image FIELD'), end=True)
        header.append(('RA', float(ra), 'Cutout centroid RA'), end=True)
        header.append(('DEC', float(dec), 'Cutout centroid DEC'), end=True)
        header.append(('COLC', colc, 'SDSS image column center'), end=True)
        header.append(('ROWC', rowc, 'SDSS image row center'), end=True)
        header.append(('GAIN', gain, 'SDSS CCD GAIN'), end=True)
        header.append(('ZERO', aa, 'SDSS image zeropoint'), end=True)
        header.append(('EXTC', kk, 'SDSS image atm. extinction coefficient'),
                      end=True)
        header.append(('AIRM', airmass, 'SDSS image airmass'), end=True)
        header.append(
            ('SKY', sky, 'Average sky in full SDSS field [nanomaggies]'),
            end=True)
        header.append(('SKYSIG', skysig,
                       'Average sky uncertainty per pixel [nanomaggies]'),
                      end=True)

    gimage = outputName
    if os.access(gimage, 0): os.remove(gimage)

    #    print('\nAfter Realism:')
    #    print('kpc_per_arcsec: {}'.format(kpc_per_arcsec))
    #    print('kpc_per_pixel: {}'.format(kpc_per_pixel))
    #    print('arcsec_per_pixel: {}'.format(arcsec_per_pixel))
    #    m_AB = -2.5*np.log10(np.sum(img_nanomaggies))+22.5
    #    print('AB_magnitude: {} at z={}'.format(m_AB,redshift))
    #    M_AB = m_AB-5*np.log10(luminosity_distance.value)-25
    #    print('AB_Magnitude: {}'.format(M_AB))

    hdu_pri = fits.PrimaryHDU(img_nanomaggies)

    header['REDSHIFT'] = (redshift, 'Redshift')
    header.append(('COSMO', 'FLAT_LCDM', 'Cosmology'), end=True)
    header.append(('OMEGA_M', cosmo.Om(0), 'Matter density'), end=True)
    header.append(('OMEGA_L', cosmo.Ode(0), 'Dark energy density'), end=True)
    header.append(('SCALE_1', arcsec_per_pixel, '[arcsec/pixel]'), end=True)
    header.append(('SCALE_2', kpc_per_pixel, '[kpc/pixel]'), end=True)
    header.append(('SCALE_3', kpc_per_arcsec, '[kpc/arcsec]'), end=True)
    header.append(('LUMDIST', cosmo.luminosity_distance(z=redshift).value,
                   'Luminosity Distance [Mpc]'),
                  end=True)
    warnings.simplefilter('ignore', category=AstropyWarning)
    header.extend(zip(common_args.keys(), common_args.values()), unique=True)
    hdu_pri.header = header
    hdu_pri.writeto(gimage)
Example #52
0
def add_sdss_stars(radeg, decdeg, out_sdss, sdss_fields):
    import sqlcl
    #lines = sqlcl.query("select ra,dec,u,g,r,i,z from star").readlines()
    ramin = (radeg - 0.33333)
    ramax = (radeg + 0.33333)
    decmin = (decdeg - 0.33333)
    decmax = (decdeg + 0.33333)
    query = "select clean, ra,dec,raErr,decErr,objID,psfMag_u,psfMag_g,psfMag_r,psfMag_i,psfMag_z,psfMagErr_u,psfMagErr_g,psfMagErr_r,psfMagErr_i,psfMagErr_z,flags_u,flags_g,flags_r,flags_i,flags_z from star where ra between " + str(
        ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(
            decmin)[:8] + " and " + str(decmax)[:8] + "    \
            AND ((flags & 0x10000000) != 0) \
                AND ((flags & 0x8100000800a4) = 0) \
            AND (((flags & 0x400000000000) = 0) or (psfmagerr_g <= 0.2)) \
               AND (((flags & 0x100000000000) = 0) or (flags & 0x1000) = 0) \
         "

    #query = "select top 10 flags_u, flags2_u from star where ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  + " "\

    #query = "select top 10 psfMagu, psfMagg, psfMagr, psfMagi, psfMagz, from star where ra between " + str(ramin)[:8] + " and  " + str(ramax)[:8] + " and  dec between " + str(decmin)[:8] + " and " +str(decmax)[:8]  + " "
    print query

    lines = sqlcl.query(query).readlines()
    uu = open('store', 'w')
    import pickle
    pickle.dump(lines, uu)

    #raw_input()
    columns = lines[0][:-1].split(',')
    print columns
    data = []
    print columns

    for line in range(1, len(lines[1:]) + 1):
        dt0 = {}
        for j in range(len(lines[line][:-1].split(','))):
            dt0[columns[j]] = lines[line][:-1].split(',')[j]
        #print line
        import string
        if string.find(lines[line][:-1], 'font') == -1:
            data.append(dt0)
        #if string.find(lines[line][:-1],'font') != -1:
        #print lines[line][:-1]
        #raw_input()

    print len(data)
    if len(data) > 0:
        sdss_fields.write(
            str(radeg) + " " + str(decdeg) + " 0.3333333 0.3333333\n")
        print len(data)

        seqnr = 1
        for els in range(len(data)):
            if 1 == 1:  #data[els].has_key('u'):

                import math

                ra = data[els]['ra']
                dec = data[els]['dec']
                u = data[els]['psfMag_u']
                g = data[els]['psfMag_g']
                r = data[els]['psfMag_r']
                i = data[els]['psfMag_i']
                z = data[els]['psfMag_z']
                uerr = data[els]['psfMagErr_u']
                gerr = data[els]['psfMagErr_g']
                rerr = data[els]['psfMagErr_r']
                ierr = data[els]['psfMagErr_i']
                zerr = data[els]['psfMagErr_z']

                vars = [ra, dec, u, g, r, i, z, uerr, gerr, rerr, ierr, zerr]
                varsstr = reduce(lambda x, y: x + ' ' + y, vars)
                out_sdss.write(varsstr + '\n')
Num = str(sys.argv[2])
lowlim = float(sys.argv[3])
highlim = float(sys.argv[4])

fullcat_name_path =  os.path.join(CURRENT_DIR,"Full_SDSS.dat")
trimmedcat_name_path = os.path.join(CURRENT_DIR,"Trimmed_SDSS.dat")
fullcat = open(fullcat_name_path,'w')
trimmedcat =open(trimmedcat_name_path,'w')

query = """
SELECT TOP """+str(Num)+"""
cast(str(p.ra,13,8) as float) as ra,cast(str(p.[dec],13,8) as float) as dec,p.psfMag_u,p.psfMag_g,p.psfMag_r,p.psfMag_i,p.psfMag_z,p.psfMagErr_u,p.psfMagErr_g,p.psfMagErr_r,p.psfMagErr_i,p.psfMagErr_z,dbo.fIAUFromEq(p.ra,p.[dec]) as SDSSname 
FROM ..PhotoObj AS p
"""+"JOIN dbo.fGetNearbyObjEq("+str(RA)+","+str(DEC)+","+str(Area)+") AS b ON b.objID = P.objID"

data = sqlcl.query(query).read()
#print data
print>>fullcat,str(data)
fullcat.close()
############################### slim down #########################

table = numpy.genfromtxt(fullcat_name_path,delimiter =',',dtype = str,skip_header = 2,unpack = True)
RA_star,DEC_star,psfMag_u,psfMag_g,psfMag_r,psfMag_i,psfMag_z,psfMagErr_u,psfMagErr_g,psfMagErr_r,psfMagErr_i,psfMagErr_z,SDSSname  = table[:]

RA_star = numpy.array(RA_star,dtype = float)
DEC_star= numpy.array(DEC_star,dtype = float)
psfMag_u= numpy.array(psfMag_u,dtype = float)
psfMag_g= numpy.array(psfMag_g,dtype = float)
psfMag_r= numpy.array(psfMag_r,dtype = float)
psfMag_i= numpy.array(psfMag_i,dtype = float)
psfMag_z= numpy.array(psfMag_z,dtype = float)
if (len(sys.argv) == 1 or len(sys.argv) != 3):
    print "It checks the SDSS PhotoObjAll catalog to find all photometric objects\n within a given radius."
    print "usage : sdss_photo_check.py (ra) (dec) -r (search radius)"
    print "\tra : degree"
    print "\tdec : degree"
    print "\tsearch radius : arcsec (optional) (default : 3 arcsec)"
    print "output : ObjId model_u model_g model_r model_i model_z"
    sys.exit()

search_rad = str(3.0 / 60.0)
args = "-r"
optlist, args = getopt.getopt(args, 'r:')
for o, a in optlist:
    if o == "-r":
        search_rad = str(float(a) / 60.0)

ra = sys.argv[1]
dec = sys.argv[2]

sql_query = "select P.objid, P.modelMag_u, P.modelMag_g, P.modelMag_r, P.modelMag_i, P.modelMag_z from PhotoObjAll P, dbo.fGetNearbyObjAllEq(" + ra + "," + dec + "," + search_rad + ") n where P.objID = n.objID"
query_result = sqlcl.query(sql_query).readlines()
if len(query_result) > 1:
    data_part = string.split(query_result[1], ",")
    output_string = ""
    for x in data_part:
        output_string = output_string + x.strip() + " "
    print output_string
    time.sleep(1.0)
else:
    print "No object found"
             join galaxy as g on s.bestObjID = g.objID where s.ra between " +\
             str(ramin) + " and " + str(ramax) + " and s.dec between " +\
             str(decmin) + " and " + str(decmax)

if objects_mode == "GALPHOT" :
    query = "select modelMag_u, modelMag_g, modelMag_r, modelMag_i, \
             modelMag_z, modelMagErr_u, modelMagErr_g, \
             modelMagErr_r, modelMagErr_i, modelMagErr_z, \
             objID, ra, dec, raErr, decErr, flags \
             from galaxy where ra between " + \
             str(ramin) + " and " + str(ramax) + " and dec between " +\
             str(decmin) + " and " + str(decmax) + \
             " AND flags & dbo.fPhotoFlags('BLENDED') = 0 "

# query the SDSS database:
lines = sqlcl.query(query,public_url).readlines()
if len(lines) == 8:
    print "An error occured during your request; probably"
    print "the selected area is too large"
    sys.exit(1)

# This became necessary due to a change in SDSS!
if catalog == "SDSSDR10":
	START=2
	columns = lines[1][:-1].split(',')
else:
	START=1
	columns = lines[0][:-1].split(',')

data = []
Example #56
0
  for i in range(count, count + queryLimit):
    ra = data[i]['RAJ2000']
    dec = data[i]['DEJ2000']
    if needOR:
      query = query + "\nOR"
    else:
      needOR = True
    query = query + " ((ra BETWEEN " + `ra - region` + " AND " + `ra + region` + ") AND (dec BETWEEN " + `dec - region` + " AND " + `dec + region` + "))"
  if printQueries:
    f1 = open("queryText/ppqueryText"+`count`+".txt", "w")
    f1.write(query)
    f1.close()
    print "written " + `count`

  if executeQuery:
    lines = sql.query(query).readlines()
    #res = []
    if lines[0][0:-1] != "#Table1":
      print "INCORRECT FORMAT RETURNED"
    else:
      extra = False
      if len(lines[2:]) > queryLimit:
        print "this query has extras"
        extra = True
      for line in lines[2:]:
        splitLine = line[:-1].split(',')
	arr = [int(splitLine[0]),float(splitLine[1]),float(splitLine[2])]
	if extra:
	  arr.append("-----")
        res.append(arr)
      if extra: