def get_cat(method,retries=100):
cwd=os.getcwd()
try:
os.mkdir(method)
except OSError:
pass
if method=='pslocal':
hplist=[]
if method=='wise':
from astroquery.irsa import Irsa
Irsa.ROW_LIMIT=1000000
ra_factor,pos=tile(cwd+'/image_ampphase1.app.restored.fits')
print 'Downloading catalogues for',len(pos),'sky positions'
for i,p in enumerate(pos):
outfile=method+'/'+method+'-'+str(i)+'.vo'
if os.path.isfile(outfile):
print 'Catalogue at position',p,'already present'
continue
print 'Downloading at position',p
if method=='panstarrs':
count=0
while True:
try:
r = requests.post('http://archive.stsci.edu/panstarrs/search.php', data = {'ra':p[0],'dec':p[1],'SR':CSIZE,'max_records':100000,'nDetections':">+5",'action':'Search','selectedColumnsCsv':'objid,ramean,decmean'},timeout=300)
except requests.exceptions.Timeout:
print 'Timeout, retrying!'
else:
if 'Warning' not in r.text and 'Please' not in r.text:
break
else:
# will go round the loop again
print 'Bad response, retry download (%i)' % count
sleep(5+count*15)
count+=1
if count>=retries:
raise RuntimeError('Number of retries exceeded for download')
f=open(outfile,'w')
f.writelines(r.text)
f.close()
elif method=='wise':
t=Irsa.query_region(coord.SkyCoord(p[0],p[1],unit=(u.deg,u.deg)), catalog='allwise_p3as_psd', radius='0d30m0s')
t.write(outfile,format='votable')
elif method=='pslocal':
from astropy_healpix import HEALPix
hp = HEALPix(nside=64)
cs = hp.cone_search_lonlat(p[0]*u.deg, p[1]*u.deg, radius=CSIZE*u.deg)
hplist += list(cs)
if not os.path.isdir(PSBASE):
# we don't have a local PS database, so download
for pix in cs:
outfile=method+'/'+str(pix)
if not os.path.isfile(outfile):
print 'Downloading healpix pixel',pix
download_file('http://uhhpc.herts.ac.uk/panstarrs-healpix/'+str(pix),outfile)
else:
raise NotImplementedError('Method '+method)
if method=='pslocal':
hplist=list(set(hplist))
print 'Found',len(hplist),'unique healpix pixels'
outname=method+'/'+method+'.txt'
with open(outname,'w') as outfile:
outfile.write('# RA DEC ObjID\n')
for pixel in hplist:
print 'Appending pixel',pixel
if os.path.isdir(PSBASE):
pixelfile=PSBASE+'/'+str(pixel)
else:
pixelfile=method+'/'+str(pixel)
if not os.path.isfile(pixelfile):
raise RuntimeError('Pixel file '+pixelfile+'does not exist')
os.system('cat '+pixelfile+' >> '+outname)
def get_cat(method, retries=100):
cwd = os.getcwd()
try:
os.mkdir(method)
except OSError:
pass
if method == 'pslocal':
hplist = []
if method == 'wise':
from astroquery.irsa import Irsa
Irsa.ROW_LIMIT = 1000000
ra_factor, pos = tile(find_fullres_image())
print 'Downloading catalogues for', len(pos), 'sky positions'
for i, p in enumerate(pos):
outfile = method + '/' + method + '-' + str(i) + '.vo'
if os.path.isfile(outfile):
print 'Catalogue at position', p, 'already present'
continue
print 'Downloading at position', p
if method == 'panstarrs':
count = 0
while True:
try:
r = requests.post(
'http://archive.stsci.edu/panstarrs/search.php',
data={
'ra': p[0],
'dec': p[1],
'SR': CSIZE,
'max_records': 100000,
'nDetections': ">+5",
'action': 'Search',
'selectedColumnsCsv': 'objid,ramean,decmean'
},
timeout=300)
except requests.exceptions.Timeout:
print 'Timeout, retrying!'
else:
if 'Warning' not in r.text and 'Please' not in r.text:
break
else:
# will go round the loop again
print 'Bad response, retry download (%i)' % count
sleep(5 + count * 15)
count += 1
if count >= retries:
raise RuntimeError(
'Number of retries exceeded for download')
f = open(outfile, 'w')
f.writelines(r.text)
f.close()
elif method == 'wise':
t = Irsa.query_region(coord.SkyCoord(p[0],
p[1],
unit=(u.deg, u.deg)),
catalog='allwise_p3as_psd',
radius='0d30m0s')
t.write(outfile, format='votable')
elif method == 'pslocal':
from astropy_healpix import HEALPix
hp = HEALPix(nside=64)
cs = hp.cone_search_lonlat(p[0] * u.deg,
p[1] * u.deg,
radius=CSIZE * u.deg)
hplist += list(cs)
if not os.path.isdir(PSBASE):
# we don't have a local PS database, so download
for pix in cs:
outfile = method + '/' + str(pix)
if not os.path.isfile(outfile):
print 'Downloading healpix pixel', pix
download_file(
'http://uhhpc.herts.ac.uk/panstarrs-healpix/' +
str(pix), outfile)
else:
raise NotImplementedError('Method ' + method)
if method == 'pslocal':
hplist = list(set(hplist))
print 'Found', len(hplist), 'unique healpix pixels'
outname = method + '/' + method + '.txt'
with open(outname, 'w') as outfile:
outfile.write('# RA DEC ObjID\n')
for pixel in hplist:
print 'Appending pixel', pixel
if os.path.isdir(PSBASE):
pixelfile = PSBASE + '/' + str(pixel)
else:
pixelfile = method + '/' + str(pixel)
if not os.path.isfile(pixelfile):
raise RuntimeError('Pixel file ' + pixelfile +
'does not exist')
os.system('cat ' + pixelfile + ' >> ' + outname)
from surveys_db import SurveysDB
from mocpy import MOC
from astropy import units as u
from astropy_healpix import HEALPix
import numpy as np
hp = HEALPix(nside=1024, order='nested')
with SurveysDB(readonly=True) as sdb:
sdb.cur.execute('select ra,decl from fields where dr2=1')
result = sdb.cur.fetchall()
ra = []
dec = []
for r in result:
ra.append(r['ra'])
dec.append(r['decl'])
plist = []
for r, d in zip(ra, dec):
pixels = hp.cone_search_lonlat(r * u.deg, d * u.deg, 1.85 * u.deg)
plist = plist + list(pixels)
p = set(plist)
pn = np.array(list(p), dtype=np.uint64)
depth = np.array([10] * len(pn), dtype=np.uint8)
moc = MOC.from_healpix_cells(pn, depth)
moc.write('/home/mjh/dr2-moc.moc', overwrite=True)
if size<60:
size=60.0
size=(int(0.5+size/10))*10
size/=3600.0
initfactor=4.0
scalefactor=initfactor
overlaygals=False
while True:
print('Scale factor is',scalefactor)
if overlaygals:
rsize=size*scalefactor/np.cos(dec*np.pi/180.0)
dsize=size*scalefactor
if dynamic_opt:
# do a cone search because looking at the corners may not work for large areas
lhp=hp.cone_search_lonlat(ra*u.deg,dec*u.deg,radius=size*u.deg)
#lhp=hp.lonlat_to_healpix(np.array([ra,ra-rsize,ra+rsize,ra-rsize,ra+rsize])*u.deg,np.array([dec,dec-dsize,dec-dsize,dec+dsize,dec+dsize])*u.deg)
lpix=sorted(list(set(lhp)))
print('Healpixes are',lpix)
tables=[]
for p in lpix:
if p in od:
tables.append(od[p])
print('Appending already loaded healpix',p)
else:
newopt=load_opt(p)
if newopt is not None:
od[p]=newopt
print('Dynamically loaded healpix',p)
tables.append(newopt)
else:
else:
print('Doing full sky area')
command = 'select ra,decl from fields where dr2>0'
with SurveysDB(readonly=True) as sdb:
sdb.cur.execute(command)
result = sdb.cur.fetchall()
print('Number of pointings is', len(result))
ra = []
dec = []
for r in result:
ra.append(r['ra'])
dec.append(r['decl'])
#ra=pos[:,0]
#dec=pos[:,1]
print('RA range is', np.min(ra), np.max(ra))
print('Dec range is', np.min(dec), np.max(dec))
plist = []
for i in range(len(ra)):
pixels = hp.cone_search_lonlat(ra[i] * u.deg, dec[i] * u.deg,
1.85 * u.deg)
plist = plist + list(pixels)
print(len(plist), 'total pixels')
print(len(set(plist)), 'total distinct pixels')
print('Area is', len(set(plist)) * area, 'sq. deg')
def _get_starting_depth(self):
def compute_angular_distance(n1, n2):
return np.arctan(np.linalg.norm(np.cross(n1, n2))/np.dot(n1, n2))
def to_xyz(lon, lat):
x = np.cos(lon) * np.cos(lat)
y = np.cos(lat) * np.sin(lon)
z = np.sin(lat)
return np.array([x, y, z], dtype=np.float64)
def max_distance_center_to_vertex(depth):
nside = 1 << depth
lat1 = np.arcsin(2 / 3.0)
lat2 = np.arcsin(1 - ((1 - 1.0/nside)**2 / 3.0))
lon1 = np.pi/(4 * nside)
lon2 = 0
# Convert lon, lat to xyz, vector
n1 = to_xyz(lon=lon1, lat=lat1)
n2 = to_xyz(lon=lon2, lat=lat2)
dist = compute_angular_distance(n1, n2)
return dist # in rad
# Get the polygon vertices as a Nx3 numpy array
# Remove the last vertex as it counts double (closed polygon)
p_vertices = np.asarray(list(self.polygon.points))[0][:-1, :]
# Get the center formed by the vertices
center = p_vertices.mean(axis=0)
# Normalize it so that it lies on the unit sphere
vector.normalize_vector(center, output=center)
center = np.asarray(center)
# Compute the maximum angular distance between the polygon vertices
# and its center. This refers to the Vector version of the
# Great-circle distance computation.
#
# See https://en.wikipedia.org/wiki/Great-circle_distance
max_d = -1
# Check if the polygon covers more than an hemisphere
covers_more_than_one_hemisphere = (self.polygon.area() > 2 * np.pi)
for vertex in p_vertices:
d = compute_angular_distance(center, vertex)
if covers_more_than_one_hemisphere:
d = np.pi - d
if d > max_d:
max_d = d
# Return the min depth so that max_d > max_center_to_vertex_ipix(depth)
depth = 0
while max_distance_center_to_vertex(depth) >= max_d:
depth = depth + 1
# Get the ipixels from astropy_healpix covering the cone of (center, radius) = (center, max_d)
lon_center, lat_center = vector.vector_to_lonlat(x=center[0], y=center[1], z=center[2], degrees=False)
hp = HEALPix(nside=(1 << depth), order='nested', frame=ICRS())
starting_iter_ipix = hp.cone_search_lonlat(lon=lon_center * u.rad, lat=lat_center * u.rad, radius=max_d * u.rad)
return depth, starting_iter_ipix
def tmatch():
maxDist = 3.0 / 60 / 60
hp = HEALPix(nside=512)
tpath = '/data/work/hjtian/QSOs_NINA.txt'
tdata00 = np.loadtxt(tpath, dtype='str') #, delimiter=' '
print(tdata00.shape)
raDec00 = tdata00[:, 0:2].astype(np.float)
tdata00Match = []
for i in range(raDec00.shape[0]):
tdata00Match.append([])
root = '/data/work/hjtian/crossmatch20200118'
destDir = '/data/work/hjtian/crossmatch20200120Rematch'
if not os.path.exists(destDir):
os.system("mkdir -p %s" % (destDir))
dirs = os.listdir(root)
tdata = np.array([])
for iii, tdir in enumerate(dirs):
tdataPath = "%s/%s" % (root, tdir)
print(tdataPath)
tds = pd.read_csv(tdataPath).to_numpy()
if tdata.shape[0] == 0:
tdata = tds
elif tdata.shape[0] > 0:
tdata = np.concatenate([tdata, tds])
print(tdata.shape)
tidxHealpix = getIdx(tdata, hp)
try:
for i in range(raDec00.shape[0]):
ra00 = raDec00[i, 0]
dec00 = raDec00[i, 1]
if i % 100000 == 0:
print(i)
hpixs = hp.cone_search_lonlat(ra00 * u.deg,
dec00 * u.deg,
radius=maxDist * u.deg)
minDist = maxDist
tmchIdx = -1
for ti in hpixs:
tposs = tidxHealpix[ti]
for tpos in tposs:
ra2 = tpos[0]
dec2 = tpos[1]
tdist = getGreatCircleDistance(ra00, dec00, ra2, dec2)
if tdist <= minDist:
#print("ra1=%f,dec1=%f, ra2=%f,dec2=%f, tdist=%f, maxDist=%f"%(ra00,dec00, ra2,dec2, tdist, maxDist))
minDist = tdist
tmchIdx = tpos[2]
if tmchIdx > -1:
if len(tdata00Match[i]) > 0:
if tdata00Match[i][0] > minDist:
tm = tdata00Match[i][0]
tm[0] = minDist
tm[1] = tdata[tmchIdx]
#tdata00Match[i].append([minDist,tdata[tmchIdx]])
else:
tdata00Match[i].append([minDist, tdata[tmchIdx]])
#break
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
#break
rst1 = []
rst2 = []
for i in range(len(tdata00Match)):
if len(tdata00Match[i]) > 0:
#print(tdata00Match[i])
rst1.append(tdata00[i])
rst2.append(tdata00Match[i][0][1])
if len(rst1) > 0:
destDir = '/data/work/hjtian/crossmatch20200120all'
if not os.path.exists(destDir):
os.system("mkdir -p %s" % (destDir))
savePath = '%s/data1_%d.csv' % (destDir, len(rst1))
print(savePath)
GPS1_df = pd.DataFrame(rst1)
GPS1_df.to_csv(savePath, index=False)
savePath = '%s/data2_%d.csv' % (destDir, len(rst2))
print(savePath)
GPS1_df = pd.DataFrame(rst2)
GPS1_df.to_csv(savePath, index=False)
def crossMatchStatistics():
gwacData = '/data/work/wj/dat'
dirs = os.listdir(gwacData)
dirs.sort()
tNames = []
tDatas = []
obsNum = 0
for iii, tdir in enumerate(dirs):
try:
if tdir.find('21450595') > 0:
obsNum = obsNum + 1
fullPath = "%s/%s" % (gwacData, tdir)
tdata00 = np.loadtxt(fullPath, dtype='str')
if tdata00.shape[0] > 10000:
tNames.append(tdir)
tDatas.append(tdata00)
print("fName is %s, num is %d" % (tdir, tdata00.shape[0]))
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
print("total file is %d, large 10000 is %s" % (obsNum, len(tNames)))
maxDist = 30 / 60 / 60
hp = HEALPix(nside=256)
for j in range(len(tNames)):
print("\n\n*****************")
print("start match %d file: %s, has %d stars" %
(j + 1, tNames[j], tDatas[j].shape[0]))
templateData = tDatas[j]
tidxs = buildIdx(hp, templateData)
mchList = []
for i in range(len(tNames)):
tdata = tDatas[i]
if i != j:
tnum00 = 0
for td in tdata:
ra1 = float(td[1])
dec1 = float(td[2])
hpixs = hp.cone_search_lonlat(ra1 * u.deg,
dec1 * u.deg,
radius=maxDist * u.deg)
isMatch = False
for ti in hpixs:
tposs = tidxs[ti]
for tpos in tposs:
ra2 = tpos[0]
dec2 = tpos[1]
try:
tdist = getGreatCircleDistance(
ra1, dec1, ra2, dec2)
if tdist <= maxDist:
isMatch = True
tnum00 = tnum00 + 1
if tnum00 < 5:
print(
"ra1=%f,dec1=%f,ra2=%f,dec2=%f,dist=%f"
% (ra1, dec1, ra2, dec2, tdist))
break
except Exception as e:
print("domatch error")
if isMatch:
break
print("%d, fName is %s, has %d stars, match num is %d" %
(i + 1, tNames[i], tdata.shape[0], tnum00))
if tnum00 > 1000:
mchList.append([i, tnum00])
print(mchList)
def templateChoseByStarMchNum():
gwacData = '/data/work/wj/dat'
dirs = os.listdir(gwacData)
dirs.sort()
tNames = []
tDatas = []
for iii, tdir in enumerate(dirs):
try:
fullPath = "%s/%s" % (gwacData, tdir)
tdata00 = np.loadtxt(fullPath, dtype='str')
if tdir.find('Field_') == 0 and tdata00.shape[0] > 10000:
tNames.append(tdir)
tDatas.append(tdata00.shape[0])
print("fName is %s, num is %d" % (tdir, tdata00.shape[0]))
except Exception as e:
print(str(e))
tstr = traceback.format_exc()
print(tstr)
skyMapName = {}
skyMapData = {}
for i in range(len(tNames)):
tkey = tNames[i][8:18]
if tkey not in skyMapName:
skyMapName[tkey] = []
skyMapData[tkey] = []
skyMapName[tkey].append(tNames[i])
skyMapData[tkey].append(tDatas[i])
allSkyDist = []
for tkey in skyMapName:
print("\n\n*****************")
print("match sky %s" % (tkey))
tNames = skyMapName[tkey]
tDatas = skyMapData[tkey]
maxNumIdx = -1
maxNum = -1
for i in range(len(tNames)):
if tDatas[i].shape[0] > maxNum:
maxNum = tDatas[i].shape[0]
maxNumIdx = i
print("maxNum is %d, fName is %s" % (maxNum, tNames[maxNumIdx]))
maxDist = 30.0 / 60 / 60
templateData = tDatas[maxNumIdx]
hp = HEALPix(nside=256)
tidxs = buildIdx(hp, templateData)
allDists1 = []
for i in range(len(tNames)):
tdata = tDatas[i]
allDists2 = []
if i != maxNumIdx and tdata.shape[0] > 10000:
for td in tdata:
#print(td)
ra1 = float(td[1])
dec1 = float(td[2])
hpixs = hp.cone_search_lonlat(ra1 * u.deg,
dec1 * u.deg,
radius=maxDist * u.deg)
#print(hpixs)
minDist = 100000
for ti in hpixs:
tposs = tidxs[ti] #tidx[tpix].append((ra[j],dec[j],j))
for tpos in tposs:
ra2 = tpos[0]
dec2 = tpos[1]
try:
tdist = getGreatCircleDistance(
ra1, dec1, ra2, dec2)
if tdist <= minDist:
minDist = tdist
except Exception as e:
print("domatch error")
#print(str(e))
#tstr = traceback.format_exc()
#print(tstr)
if minDist <= maxDist:
allDists2.append(minDist)
print("fName is %s, has %d stars, match num is %d" %
(tNames[i], tdata.shape[0], len(allDists2)))
allDists1.append(allDists2)
allSkyDist.append(allDists1)
for j, tdists in enumerate(allSkyDist):
maxNum = 0
maxIdx = -1
for i, tds in enumerate(tdists):
if len(tds) > maxNum:
maxNum = len(tds)
maxIdx = i
if maxIdx > -1:
print((maxIdx, maxNum))
x = np.array(tdists[maxIdx]) * 3600
bins = np.arange(0, 30, 1)
plt.hist(x, bins, color='fuchsia', alpha=0.5)
plt.xlabel('distance')
plt.ylabel('count')
plt.grid()
plt.xlim(0, 30)
plt.show()
