def get_cat(method,retries=100): cwd=os.getcwd() try: os.mkdir(method) except OSError: pass if method=='pslocal': hplist=[] if method=='wise': from astroquery.irsa import Irsa Irsa.ROW_LIMIT=1000000 ra_factor,pos=tile(cwd+'/image_ampphase1.app.restored.fits') print 'Downloading catalogues for',len(pos),'sky positions' for i,p in enumerate(pos): outfile=method+'/'+method+'-'+str(i)+'.vo' if os.path.isfile(outfile): print 'Catalogue at position',p,'already present' continue print 'Downloading at position',p if method=='panstarrs': count=0 while True: try: r = requests.post('http://archive.stsci.edu/panstarrs/search.php', data = {'ra':p[0],'dec':p[1],'SR':CSIZE,'max_records':100000,'nDetections':">+5",'action':'Search','selectedColumnsCsv':'objid,ramean,decmean'},timeout=300) except requests.exceptions.Timeout: print 'Timeout, retrying!' else: if 'Warning' not in r.text and 'Please' not in r.text: break else: # will go round the loop again print 'Bad response, retry download (%i)' % count sleep(5+count*15) count+=1 if count>=retries: raise RuntimeError('Number of retries exceeded for download') f=open(outfile,'w') f.writelines(r.text) f.close() elif method=='wise': t=Irsa.query_region(coord.SkyCoord(p[0],p[1],unit=(u.deg,u.deg)), catalog='allwise_p3as_psd', radius='0d30m0s') t.write(outfile,format='votable') elif method=='pslocal': from astropy_healpix import HEALPix hp = HEALPix(nside=64) cs = hp.cone_search_lonlat(p[0]*u.deg, p[1]*u.deg, radius=CSIZE*u.deg) hplist += list(cs) if not os.path.isdir(PSBASE): # we don't have a local PS database, so download for pix in cs: outfile=method+'/'+str(pix) if not os.path.isfile(outfile): print 'Downloading healpix pixel',pix download_file('http://uhhpc.herts.ac.uk/panstarrs-healpix/'+str(pix),outfile) else: raise NotImplementedError('Method '+method) if method=='pslocal': hplist=list(set(hplist)) print 'Found',len(hplist),'unique healpix pixels' outname=method+'/'+method+'.txt' with open(outname,'w') as outfile: outfile.write('# RA DEC ObjID\n') for pixel in hplist: print 'Appending pixel',pixel if os.path.isdir(PSBASE): pixelfile=PSBASE+'/'+str(pixel) else: pixelfile=method+'/'+str(pixel) if not os.path.isfile(pixelfile): raise RuntimeError('Pixel file '+pixelfile+'does not exist') os.system('cat '+pixelfile+' >> '+outname)
def get_cat(method, retries=100): cwd = os.getcwd() try: os.mkdir(method) except OSError: pass if method == 'pslocal': hplist = [] if method == 'wise': from astroquery.irsa import Irsa Irsa.ROW_LIMIT = 1000000 ra_factor, pos = tile(find_fullres_image()) print 'Downloading catalogues for', len(pos), 'sky positions' for i, p in enumerate(pos): outfile = method + '/' + method + '-' + str(i) + '.vo' if os.path.isfile(outfile): print 'Catalogue at position', p, 'already present' continue print 'Downloading at position', p if method == 'panstarrs': count = 0 while True: try: r = requests.post( 'http://archive.stsci.edu/panstarrs/search.php', data={ 'ra': p[0], 'dec': p[1], 'SR': CSIZE, 'max_records': 100000, 'nDetections': ">+5", 'action': 'Search', 'selectedColumnsCsv': 'objid,ramean,decmean' }, timeout=300) except requests.exceptions.Timeout: print 'Timeout, retrying!' else: if 'Warning' not in r.text and 'Please' not in r.text: break else: # will go round the loop again print 'Bad response, retry download (%i)' % count sleep(5 + count * 15) count += 1 if count >= retries: raise RuntimeError( 'Number of retries exceeded for download') f = open(outfile, 'w') f.writelines(r.text) f.close() elif method == 'wise': t = Irsa.query_region(coord.SkyCoord(p[0], p[1], unit=(u.deg, u.deg)), catalog='allwise_p3as_psd', radius='0d30m0s') t.write(outfile, format='votable') elif method == 'pslocal': from astropy_healpix import HEALPix hp = HEALPix(nside=64) cs = hp.cone_search_lonlat(p[0] * u.deg, p[1] * u.deg, radius=CSIZE * u.deg) hplist += list(cs) if not os.path.isdir(PSBASE): # we don't have a local PS database, so download for pix in cs: outfile = method + '/' + str(pix) if not os.path.isfile(outfile): print 'Downloading healpix pixel', pix download_file( 'http://uhhpc.herts.ac.uk/panstarrs-healpix/' + str(pix), outfile) else: raise NotImplementedError('Method ' + method) if method == 'pslocal': hplist = list(set(hplist)) print 'Found', len(hplist), 'unique healpix pixels' outname = method + '/' + method + '.txt' with open(outname, 'w') as outfile: outfile.write('# RA DEC ObjID\n') for pixel in hplist: print 'Appending pixel', pixel if os.path.isdir(PSBASE): pixelfile = PSBASE + '/' + str(pixel) else: pixelfile = method + '/' + str(pixel) if not os.path.isfile(pixelfile): raise RuntimeError('Pixel file ' + pixelfile + 'does not exist') os.system('cat ' + pixelfile + ' >> ' + outname)
from surveys_db import SurveysDB from mocpy import MOC from astropy import units as u from astropy_healpix import HEALPix import numpy as np hp = HEALPix(nside=1024, order='nested') with SurveysDB(readonly=True) as sdb: sdb.cur.execute('select ra,decl from fields where dr2=1') result = sdb.cur.fetchall() ra = [] dec = [] for r in result: ra.append(r['ra']) dec.append(r['decl']) plist = [] for r, d in zip(ra, dec): pixels = hp.cone_search_lonlat(r * u.deg, d * u.deg, 1.85 * u.deg) plist = plist + list(pixels) p = set(plist) pn = np.array(list(p), dtype=np.uint64) depth = np.array([10] * len(pn), dtype=np.uint8) moc = MOC.from_healpix_cells(pn, depth) moc.write('/home/mjh/dr2-moc.moc', overwrite=True)
if size<60: size=60.0 size=(int(0.5+size/10))*10 size/=3600.0 initfactor=4.0 scalefactor=initfactor overlaygals=False while True: print('Scale factor is',scalefactor) if overlaygals: rsize=size*scalefactor/np.cos(dec*np.pi/180.0) dsize=size*scalefactor if dynamic_opt: # do a cone search because looking at the corners may not work for large areas lhp=hp.cone_search_lonlat(ra*u.deg,dec*u.deg,radius=size*u.deg) #lhp=hp.lonlat_to_healpix(np.array([ra,ra-rsize,ra+rsize,ra-rsize,ra+rsize])*u.deg,np.array([dec,dec-dsize,dec-dsize,dec+dsize,dec+dsize])*u.deg) lpix=sorted(list(set(lhp))) print('Healpixes are',lpix) tables=[] for p in lpix: if p in od: tables.append(od[p]) print('Appending already loaded healpix',p) else: newopt=load_opt(p) if newopt is not None: od[p]=newopt print('Dynamically loaded healpix',p) tables.append(newopt) else:
else: print('Doing full sky area') command = 'select ra,decl from fields where dr2>0' with SurveysDB(readonly=True) as sdb: sdb.cur.execute(command) result = sdb.cur.fetchall() print('Number of pointings is', len(result)) ra = [] dec = [] for r in result: ra.append(r['ra']) dec.append(r['decl']) #ra=pos[:,0] #dec=pos[:,1] print('RA range is', np.min(ra), np.max(ra)) print('Dec range is', np.min(dec), np.max(dec)) plist = [] for i in range(len(ra)): pixels = hp.cone_search_lonlat(ra[i] * u.deg, dec[i] * u.deg, 1.85 * u.deg) plist = plist + list(pixels) print(len(plist), 'total pixels') print(len(set(plist)), 'total distinct pixels') print('Area is', len(set(plist)) * area, 'sq. deg')
def _get_starting_depth(self): def compute_angular_distance(n1, n2): return np.arctan(np.linalg.norm(np.cross(n1, n2))/np.dot(n1, n2)) def to_xyz(lon, lat): x = np.cos(lon) * np.cos(lat) y = np.cos(lat) * np.sin(lon) z = np.sin(lat) return np.array([x, y, z], dtype=np.float64) def max_distance_center_to_vertex(depth): nside = 1 << depth lat1 = np.arcsin(2 / 3.0) lat2 = np.arcsin(1 - ((1 - 1.0/nside)**2 / 3.0)) lon1 = np.pi/(4 * nside) lon2 = 0 # Convert lon, lat to xyz, vector n1 = to_xyz(lon=lon1, lat=lat1) n2 = to_xyz(lon=lon2, lat=lat2) dist = compute_angular_distance(n1, n2) return dist # in rad # Get the polygon vertices as a Nx3 numpy array # Remove the last vertex as it counts double (closed polygon) p_vertices = np.asarray(list(self.polygon.points))[0][:-1, :] # Get the center formed by the vertices center = p_vertices.mean(axis=0) # Normalize it so that it lies on the unit sphere vector.normalize_vector(center, output=center) center = np.asarray(center) # Compute the maximum angular distance between the polygon vertices # and its center. This refers to the Vector version of the # Great-circle distance computation. # # See https://en.wikipedia.org/wiki/Great-circle_distance max_d = -1 # Check if the polygon covers more than an hemisphere covers_more_than_one_hemisphere = (self.polygon.area() > 2 * np.pi) for vertex in p_vertices: d = compute_angular_distance(center, vertex) if covers_more_than_one_hemisphere: d = np.pi - d if d > max_d: max_d = d # Return the min depth so that max_d > max_center_to_vertex_ipix(depth) depth = 0 while max_distance_center_to_vertex(depth) >= max_d: depth = depth + 1 # Get the ipixels from astropy_healpix covering the cone of (center, radius) = (center, max_d) lon_center, lat_center = vector.vector_to_lonlat(x=center[0], y=center[1], z=center[2], degrees=False) hp = HEALPix(nside=(1 << depth), order='nested', frame=ICRS()) starting_iter_ipix = hp.cone_search_lonlat(lon=lon_center * u.rad, lat=lat_center * u.rad, radius=max_d * u.rad) return depth, starting_iter_ipix
def tmatch(): maxDist = 3.0 / 60 / 60 hp = HEALPix(nside=512) tpath = '/data/work/hjtian/QSOs_NINA.txt' tdata00 = np.loadtxt(tpath, dtype='str') #, delimiter=' ' print(tdata00.shape) raDec00 = tdata00[:, 0:2].astype(np.float) tdata00Match = [] for i in range(raDec00.shape[0]): tdata00Match.append([]) root = '/data/work/hjtian/crossmatch20200118' destDir = '/data/work/hjtian/crossmatch20200120Rematch' if not os.path.exists(destDir): os.system("mkdir -p %s" % (destDir)) dirs = os.listdir(root) tdata = np.array([]) for iii, tdir in enumerate(dirs): tdataPath = "%s/%s" % (root, tdir) print(tdataPath) tds = pd.read_csv(tdataPath).to_numpy() if tdata.shape[0] == 0: tdata = tds elif tdata.shape[0] > 0: tdata = np.concatenate([tdata, tds]) print(tdata.shape) tidxHealpix = getIdx(tdata, hp) try: for i in range(raDec00.shape[0]): ra00 = raDec00[i, 0] dec00 = raDec00[i, 1] if i % 100000 == 0: print(i) hpixs = hp.cone_search_lonlat(ra00 * u.deg, dec00 * u.deg, radius=maxDist * u.deg) minDist = maxDist tmchIdx = -1 for ti in hpixs: tposs = tidxHealpix[ti] for tpos in tposs: ra2 = tpos[0] dec2 = tpos[1] tdist = getGreatCircleDistance(ra00, dec00, ra2, dec2) if tdist <= minDist: #print("ra1=%f,dec1=%f, ra2=%f,dec2=%f, tdist=%f, maxDist=%f"%(ra00,dec00, ra2,dec2, tdist, maxDist)) minDist = tdist tmchIdx = tpos[2] if tmchIdx > -1: if len(tdata00Match[i]) > 0: if tdata00Match[i][0] > minDist: tm = tdata00Match[i][0] tm[0] = minDist tm[1] = tdata[tmchIdx] #tdata00Match[i].append([minDist,tdata[tmchIdx]]) else: tdata00Match[i].append([minDist, tdata[tmchIdx]]) #break except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) #break rst1 = [] rst2 = [] for i in range(len(tdata00Match)): if len(tdata00Match[i]) > 0: #print(tdata00Match[i]) rst1.append(tdata00[i]) rst2.append(tdata00Match[i][0][1]) if len(rst1) > 0: destDir = '/data/work/hjtian/crossmatch20200120all' if not os.path.exists(destDir): os.system("mkdir -p %s" % (destDir)) savePath = '%s/data1_%d.csv' % (destDir, len(rst1)) print(savePath) GPS1_df = pd.DataFrame(rst1) GPS1_df.to_csv(savePath, index=False) savePath = '%s/data2_%d.csv' % (destDir, len(rst2)) print(savePath) GPS1_df = pd.DataFrame(rst2) GPS1_df.to_csv(savePath, index=False)
def crossMatchStatistics(): gwacData = '/data/work/wj/dat' dirs = os.listdir(gwacData) dirs.sort() tNames = [] tDatas = [] obsNum = 0 for iii, tdir in enumerate(dirs): try: if tdir.find('21450595') > 0: obsNum = obsNum + 1 fullPath = "%s/%s" % (gwacData, tdir) tdata00 = np.loadtxt(fullPath, dtype='str') if tdata00.shape[0] > 10000: tNames.append(tdir) tDatas.append(tdata00) print("fName is %s, num is %d" % (tdir, tdata00.shape[0])) except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) print("total file is %d, large 10000 is %s" % (obsNum, len(tNames))) maxDist = 30 / 60 / 60 hp = HEALPix(nside=256) for j in range(len(tNames)): print("\n\n*****************") print("start match %d file: %s, has %d stars" % (j + 1, tNames[j], tDatas[j].shape[0])) templateData = tDatas[j] tidxs = buildIdx(hp, templateData) mchList = [] for i in range(len(tNames)): tdata = tDatas[i] if i != j: tnum00 = 0 for td in tdata: ra1 = float(td[1]) dec1 = float(td[2]) hpixs = hp.cone_search_lonlat(ra1 * u.deg, dec1 * u.deg, radius=maxDist * u.deg) isMatch = False for ti in hpixs: tposs = tidxs[ti] for tpos in tposs: ra2 = tpos[0] dec2 = tpos[1] try: tdist = getGreatCircleDistance( ra1, dec1, ra2, dec2) if tdist <= maxDist: isMatch = True tnum00 = tnum00 + 1 if tnum00 < 5: print( "ra1=%f,dec1=%f,ra2=%f,dec2=%f,dist=%f" % (ra1, dec1, ra2, dec2, tdist)) break except Exception as e: print("domatch error") if isMatch: break print("%d, fName is %s, has %d stars, match num is %d" % (i + 1, tNames[i], tdata.shape[0], tnum00)) if tnum00 > 1000: mchList.append([i, tnum00]) print(mchList)
def templateChoseByStarMchNum(): gwacData = '/data/work/wj/dat' dirs = os.listdir(gwacData) dirs.sort() tNames = [] tDatas = [] for iii, tdir in enumerate(dirs): try: fullPath = "%s/%s" % (gwacData, tdir) tdata00 = np.loadtxt(fullPath, dtype='str') if tdir.find('Field_') == 0 and tdata00.shape[0] > 10000: tNames.append(tdir) tDatas.append(tdata00.shape[0]) print("fName is %s, num is %d" % (tdir, tdata00.shape[0])) except Exception as e: print(str(e)) tstr = traceback.format_exc() print(tstr) skyMapName = {} skyMapData = {} for i in range(len(tNames)): tkey = tNames[i][8:18] if tkey not in skyMapName: skyMapName[tkey] = [] skyMapData[tkey] = [] skyMapName[tkey].append(tNames[i]) skyMapData[tkey].append(tDatas[i]) allSkyDist = [] for tkey in skyMapName: print("\n\n*****************") print("match sky %s" % (tkey)) tNames = skyMapName[tkey] tDatas = skyMapData[tkey] maxNumIdx = -1 maxNum = -1 for i in range(len(tNames)): if tDatas[i].shape[0] > maxNum: maxNum = tDatas[i].shape[0] maxNumIdx = i print("maxNum is %d, fName is %s" % (maxNum, tNames[maxNumIdx])) maxDist = 30.0 / 60 / 60 templateData = tDatas[maxNumIdx] hp = HEALPix(nside=256) tidxs = buildIdx(hp, templateData) allDists1 = [] for i in range(len(tNames)): tdata = tDatas[i] allDists2 = [] if i != maxNumIdx and tdata.shape[0] > 10000: for td in tdata: #print(td) ra1 = float(td[1]) dec1 = float(td[2]) hpixs = hp.cone_search_lonlat(ra1 * u.deg, dec1 * u.deg, radius=maxDist * u.deg) #print(hpixs) minDist = 100000 for ti in hpixs: tposs = tidxs[ti] #tidx[tpix].append((ra[j],dec[j],j)) for tpos in tposs: ra2 = tpos[0] dec2 = tpos[1] try: tdist = getGreatCircleDistance( ra1, dec1, ra2, dec2) if tdist <= minDist: minDist = tdist except Exception as e: print("domatch error") #print(str(e)) #tstr = traceback.format_exc() #print(tstr) if minDist <= maxDist: allDists2.append(minDist) print("fName is %s, has %d stars, match num is %d" % (tNames[i], tdata.shape[0], len(allDists2))) allDists1.append(allDists2) allSkyDist.append(allDists1) for j, tdists in enumerate(allSkyDist): maxNum = 0 maxIdx = -1 for i, tds in enumerate(tdists): if len(tds) > maxNum: maxNum = len(tds) maxIdx = i if maxIdx > -1: print((maxIdx, maxNum)) x = np.array(tdists[maxIdx]) * 3600 bins = np.arange(0, 30, 1) plt.hist(x, bins, color='fuchsia', alpha=0.5) plt.xlabel('distance') plt.ylabel('count') plt.grid() plt.xlim(0, 30) plt.show()