def get_fitting_region(order, pixel):
"""Expand tile by quarter of a pixel for fitting
:param order: the HEALPix resolution level
:param pixel: given HEALPix pixel that needs to be fit
:return: HEALPix pixels that need to be fit
"""
#define old and new order
old_nside = 2**order
new_nside = 2**(order + 2)
#get co-ord of main pixel
theta, phi = pixelfunc.pix2ang(old_nside, pixel, nest=True)
#define offsets such that main pixel is split into four sub pixels
scale = pixelfunc.max_pixrad(old_nside)
offset_theta = np.array([-0.125, 0.0, 0.125, 0.0]) * scale
offset_phi = np.array([0.0, -0.125, 0.0, 0.125]) * scale
#convert co-ords to pixels at higher order
pix_fit = pixelfunc.ang2pix(new_nside,
theta + offset_theta,
phi + offset_phi,
nest=True)
#get neighbouring pixels and remove duplicates
moc_tile = MOC()
pixels = np.unique(
pixelfunc.get_all_neighbours(new_nside, pix_fit, nest=True))
moc_tile.add(
order + 2,
np.unique(pixelfunc.get_all_neighbours(new_nside, pixels, nest=True)))
return moc_tile
def healpix_bins(nside, rs, phis, thetas):
npix = pf.nside2npix(nside)
ipix = pf.ang2pix(nside, thetas, phis)
r_bins = [[] for _ in xrange(npix)]
for (r, i) in zip(rs, ipix):
r_bins[i].append(r)
return r_bins
def healpix_bins(nside, rs, phis, thetas):
npix = pf.nside2npix(nside)
ipix = pf.ang2pix(nside, thetas, phis)
r_bins = [[] for _ in xrange(npix)]
for (r, i) in zip(rs, ipix):
r_bins[i].append(r)
return r_bins
def get_fitting_region(order,pixel):
"""Expand tile by quarter of a pixel for fitting
:param order: the HEALPix resolution level
:param pixel: given HEALPix pixel that needs to be fit
:return: HEALPix pixels that need to be fit
"""
#define old and new order
old_nside=2**order
new_nside=2**(order+2)
#get co-ord of main pixel
theta,phi=pixelfunc.pix2ang(old_nside, pixel, nest=True)
#define offsets such that main pixel is split into four sub pixels
scale=pixelfunc.max_pixrad(old_nside)
offset_theta=np.array([-0.125,0.0,0.125,0.0])*scale
offset_phi=np.array([0.0,-0.125,0.0,0.125])*scale
#convert co-ords to pixels at higher order
pix_fit=pixelfunc.ang2pix(new_nside, theta+offset_theta, phi+offset_phi, nest=True)
#get neighbouring pixels and remove duplicates
moc_tile=MOC()
pixels=np.unique(pixelfunc.get_all_neighbours(new_nside, pix_fit,nest=True))
moc_tile.add(order+2,np.unique(pixelfunc.get_all_neighbours(new_nside, pixels,nest=True)))
return moc_tile
def filter_table(self, table, ra_column, dec_column, keep_inside=True):
"""
Filter an astropy.table.Table to keep only rows inside (or outside) the MOC instance
Return the (newly created) filtered Table
"""
from healpy import pixelfunc
filtered_table = Table()
kept_rows = []
pixels_best_res = set()
for val in self.best_res_pixels_iterator():
pixels_best_res.add(val)
max_order = self.max_order
nside = 2**max_order
for row in table:
theta, phi = utils.radec2thetaphi(row[ra_column], row[dec_column])
ipix = pixelfunc.ang2pix(nside, theta, phi, nest=True)
if (ipix in pixels_best_res) == keep_inside:
kept_rows.append(row)
if len(kept_rows) == 0:
return Table(names=table.colnames)
else:
return Table(rows=kept_rows, names=table.colnames)
def add_position(self, ra, dec, max_norder):
"""
add the HEALPix bin containing the (ra, dec) position
"""
from healpy import pixelfunc
theta, phi = utils.radec2thetaphi(ra, dec)
ipix = pixelfunc.ang2pix(2**max_norder, theta, phi, nest=True)
self.add_pix(max_norder, ipix)
def skycoord_to_healpix(skycoord, nside=None):
"""
Convert an array of RA DEC values to their HEALPIX values.
"""
if nside is None:
nside = HEALPIX_NSIDE
return pixelfunc.ang2pix(nside,
skycoord.ra.degree,
skycoord.dec.degree,
lonlat=True)
def tile_in_tile(order_small, tile_small, order_large):
"""Routine to find our what larger tile to load data from when fitting smaller tiles.
Returns larger tile no. Useful for when fitting segmented maps on HPC using a hierarchical segmentation scheme
:param order_small: order of smaller tile
:param tile_small: tile no. of smaller tile
:param order_large: order of larger tiling scheme
:return: pixel number of larger tile
"""
theta, phi = pixelfunc.pix2ang(2**order_small, tile_small, nest=True)
ipix = pixelfunc.ang2pix(2**order_large, theta, phi, nest=True)
return ipix
def tile_in_tile(order_small,tile_small,order_large):
"""Routine to find our what larger tile to load data from when fitting smaller tiles.
Returns larger tile no. Useful for when fitting segmented maps on HPC using a hierarchical segmentation scheme
:param order_small: order of smaller tile
:param tile_small: tile no. of smaller tile
:param order_large: order of larger tiling scheme
:return: pixel number of larger tile
"""
theta, phi =pixelfunc.pix2ang(2**order_small, tile_small, nest=True)
ipix = pixelfunc.ang2pix(2**order_large, theta, phi, nest=True)
return ipix
def rectangle_to_healpix(nside, nest, *rectangle):
"""
Generate a unique list of HEALPix pixels for the given rectangle.
"""
size = nside2resol(nside) * 180 / pi
ra, dec = rectangle_mesh(size / size_factor, *rectangle)
# Change from RA, DEC to theta, phi where 0 <= theta <= pi
return np.unique(ang2pix(nside, pi / 2 - dec * pi / 180,
ra * pi / 180, nest=nest))
def main(pathfilein, energy, costheta, momcntrx, momcntry, galoff, calonly, tol, maxnum, output):
aHpxGalOFF = None
nhpside = 16
if galoff is True:
pathCatalogue = "/nfs/farm/g/glast/u/mtakahas/FermiAnalysis/Catalogues/gll_psch_v13.fit" #3FHL
aHpxGalOFF = find_galoff_healpxs(nhpside, 0, pathCatalogue)
print 'Galactic OFF:', aHpxGalOFF
with open(pathfilein, 'r') as inp:
print 'Input file', pathfilein, 'is opened.'
line = inp.readline()
with open(output, 'w') as ou:
print 'Output file', output, 'is opened.'
ou.write("# RunID EvtID RECON_File MERIT_File")
ncount = 0
flag_break = False
while line and flag_break is False:
line.decode('unicode-escape')
merit = line[:-1]
print merit
fileIn = ROOT.TFile.Open(merit)
print fileIn.GetName(), 'is opened.'
trIn = fileIn.Get('MeritTuple')
print trIn.GetName(), 'is found.'
for iEvt in range(trIn.GetEntries()):
trIn.GetEntry(iEvt)
if energy>=trIn.WP8CalOnlyEnergy*(1.-tol) and energy<trIn.WP8CalOnlyEnergy*(1.+tol) and costheta>=trIn.Cal1MomZDir*(1.-0.1) and costheta<trIn.Cal1MomZDir*(1.+0.1) and ( ( momcntrx is None or (abs(momcntrx)>=abs(trIn.Cal1MomXCntrCor)*(1.-tol) and abs(momcntrx)<abs(trIn.Cal1MomXCntrCor)*(1.+tol)) ) or ( momcntry is None or (abs(momcntry)>=abs(trIn.Cal1MomYCntrCor)*(1.-tol) and abs(momcntry)<abs(trIn.Cal1MomYCntrCor)*(1.+tol)) ) ) and trIn.FT1CalZenithTheta<=90.:
if calonly is False or (trIn.TkrNumTracks==0 or (math.log10(max(trIn.CalTrackAngle,1E-4)) > (0.529795)*(trIn.EvtJointLogEnergy < 3.000000) + ((1.0)*((0.529795)*(1.0)+(-1.379791)*(pow((trIn.EvtJointLogEnergy-3.000000)/0.916667,1))+(0.583401)*(pow((trIn.EvtJointLogEnergy-3.000000)/0.916667,2))+(-0.075555)*(pow((trIn.EvtJointLogEnergy-3.000000)/0.916667,3))))*(trIn.EvtJointLogEnergy >= 3.000000 and trIn.EvtJointLogEnergy <= 5.750000) + (-0.398962)*(trIn.EvtJointLogEnergy > 5.750000)) ):
if aHpxGalOFF is not None:
nhpx_evt = hppf.ang2pix(nhpside, math.pi/2.-math.radians(trIn.FT1CalDec), math.radians(trIn.FT1CalRa))
print nhpx_evt
if nhpx_evt in aHpxGalOFF:
ou.write("""
{run} {evt} {reco} {meri}""".format(run=trIn.EvtRun, evt=trIn.EvtEventId, reco=merit.replace('merit', 'recon'), meri=merit))
print ncount
ncount += 1
else:
ou.write("""
{run} {evt} {reco} {meri}""".format(run=trIn.EvtRun, evt=trIn.EvtEventId, reco=merit.replace('merit', 'recon'), meri=merit))
print ncount, 'events found.'
ncount += 1
if ncount>maxnum:
print 'Number of selected events reached {0}.'.format(ncount)
flag_break = True
break
line = inp.readline()
print ''
print 'Finished.'
def azel_value(self, az=None, el=None, n=100):
""" Get the :attr:`~nenupy.astro.hpxsky.HpxSky.skymap`
values at ``az``, ``el`` coordinates.
:param az:
Azimuth in horizontal coordinates (degrees)
Default: None
:type az: `float`, :class:`~numpy.ndarray`, or :class:`~astropy.units.Quantity`
:param el:
Elevation in horizontal coordinates (degrees)
Default: None
:type el: `float`, :class:`~numpy.ndarray`, or :class:`~astropy.units.Quantity`
:param n:
Number of points to evaluate if one coordinate is `None`
Default: 100
:type n: `int`
:returns: Sky map values at ``az``, ``el``
:rtype: :class:`~numpy.ndarray`
"""
if (az is not None) and (el is not None):
pass
elif (az is not None) and (el is None):
if isinstance(az, u.Quantity):
az = az.to(u.deg).value
az = np.ones(n) * az
el = np.linspace(0, 90, n)
elif (az is None) and (el is not None):
if isinstance(el, u.Quantity):
el = el.to(u.deg).value
az = np.linspace(0, 360, n)
el = np.ones(n) * el
else:
raise Exception('Give at least one coordinate')
# Transform to RADEC
altaz = AltAz(az=az * u.deg,
alt=el * u.deg,
location=nenufar_loc,
obstime=self.time)
radec = altaz.transform_to(ICRS)
# Get the indices
indices = ang2pix(theta=radec.ra.deg,
phi=radec.dec.deg,
nside=self.nside,
lonlat=True)
return self.skymap[indices]
def get_HEALPix_pixels(order, ra, dec, unique=True):
"""Work out what HEALPix a source is in
:param order: order of HEALPix
:param ra: Right Ascension
:param dec: Declination
:param unique: if unique is true, removes duplicate pixels
:return: list of HEALPix pixels :rtype:
"""
HPX_D2R = np.pi / 180.0
#Convert catalogue to polar co-ords in radians
phi = ra * HPX_D2R
theta = np.pi / 2.0 - dec * HPX_D2R
#calculate what pixel each object is in
ipix = pixelfunc.ang2pix(2**order, theta, phi, nest=True)
#return unique pixels (i.e. remove duplicates)
if unique is True:
return np.unique(ipix)
else:
return ipix
def get_HEALPix_pixels(order,ra,dec,unique=True):
"""Work out what HEALPix a source is in
:param order: order of HEALPix
:param ra: Right Ascension
:param dec: Declination
:param unique: if unique is true, removes duplicate pixels
:return: list of HEALPix pixels :rtype:
"""
HPX_D2R=np.pi/180.0
#Convert catalogue to polar co-ords in radians
phi = ra*HPX_D2R
theta = np.pi/2.0 - dec*HPX_D2R
#calculate what pixel each object is in
ipix = pixelfunc.ang2pix(2**order, theta, phi, nest=True)
#return unique pixels (i.e. remove duplicates)
if unique is True:
return np.unique(ipix)
else:
return ipix
def fits2csv(fits_name):
import numpy as np
import pandas as pd
from astropy import table
from astropy.io import fits
import healpy.pixelfunc as pf
hdulist = fits.open(fits_name)
tbdata = hdulist[1].data
ra = tbdata['RA']
dec = tbdata['DEC']
z = tbdata['Z']
absmag = tbdata['AMAG']
appmag = tbdata['OMAG']
d = {
'zobs': z.astype('float64'),
'ra': ra.astype('float64'),
'dec': dec.astype('float64'),
'u_ab': absmag[:, 0].astype('float64'),
'g_ab': absmag[:, 1].astype('float64'),
'r_ab': absmag[:, 2].astype('float64'),
'i_ab': absmag[:, 3].astype('float64'),
'z_ab': absmag[:, 4].astype('float64'),
'u_ap': appmag[:, 0].astype('float64'),
'g_ap': appmag[:, 1].astype('float64'),
'r_ap': appmag[:, 2].astype('float64'),
'i_ap': appmag[:, 3].astype('float64'),
'z_ap': appmag[:, 4].astype('float64')
}
fit_table = table.Table(d)
data = fit_table.to_pandas()
hdulist.close()
data['big_pix'] = pf.ang2pix(32,
theta=data['ra'],
phi=data['dec'],
lonlat=True,
nest=False)
return (data)
def desi(in_path, out_path):
import time
import numpy as np
import pandas as pd
import healpy.pixelfunc as pf
####################
# Reading CSV
####################
t0 = time.time()
var_names = [
'id_galaxy_sam', 'id_galaxy_sky', 'zcos', 'zobs', 'BCDM', 'ra', 'dec',
'u_ap', 'u_ab', 'g_ap', 'g_ab', 'r_ap', 'r_ab', 'i_ap', 'i_ab', 'z_ap',
'z_ab'
]
data = pd.read_csv(f'{in_path}DESI_LC.csv', header=None, names=var_names)
data = data.drop(columns=['id_galaxy_sam', 'zcos', 'BCDM'])
big_pixels = pf.ang2pix(32,
data['ra'],
data['dec'],
nest=False,
lonlat=True)
big_pixel_ID = np.unique(big_pixels)
####################
# Saving CSV
####################
fnames = []
for pix in big_pixel_ID:
fname = f'{out_path}galaxies_GALFORM_{pix}.csv'
fnames.append(fname)
data_subset = data.loc[big_pixels == pix]
data_subset.to_csv(fname, index=False)
t1 = time.time()
print(f'Data prepared in {t1-t0} s')
return (fnames)
def radec_value(self, ra=None, dec=None, n=100):
""" Get the :attr:`~nenupy.astro.hpxsky.HpxSky.skymap`
values at ``ra``, ``dec`` coordinates.
:param ra:
RA in equatorial coordinates (in degrees if `float`)
Default: None
:type ra: `float`, :class:`~numpy.ndarray`, or :class:`~astropy.units.Quantity`
:param dec:
Declination in equatorial coordinates (in degrees if `float`)
Default: None
:type dec: `float`, :class:`~numpy.ndarray`, or :class:`~astropy.units.Quantity`
:param n:
Number of points to evaluate if one coordinate is `None`
Default: 100
:type n: `int`
:returns: Sky map values at ``ra``, ``dec``
:rtype: :class:`~numpy.ndarray`
"""
if (ra is not None) and (dec is not None):
pass
elif (ra is not None) and (dec is None):
if isinstance(ra, u.Quantity):
ra = ra.to(u.deg).value
ra = np.ones(n) * ra
dec = np.linspace(0, 90, n)
elif (ra is None) and (dec is not None):
if isinstance(dec, u.Quantity):
dec = dec.to(u.deg).value
ra = np.linspace(0, 360, n)
dec = np.ones(n) * dec
else:
raise Exception('Give at least one coordinate')
# Get the indices
indices = ang2pix(theta=ra, phi=dec, nside=self.nside, lonlat=True)
return self.skymap[indices]
def source_positions_ang(self, nbar):
dx = self._healpix_deltax+1
# Limit the range on the sky that we populate for efficiency
maxtheta = self.get_maxtheta()
angular_size = 2*np.pi*(1 - np.cos(maxtheta))
ntot = np.random.poisson(nbar * angular_size)
pos = np.zeros((2,ntot))
ndone = 0
nleft = ntot
pixweights = dx / np.max(dx)
overweight = len(pixweights)/np.sum(pixweights)
while nleft > 0:
theta, phi = randsphere(int(nleft*overweight), theta_range=(0,maxtheta))
pix = pixelfunc.ang2pix(self.params['nside'], theta, phi)
weights = pixweights[pix]
rnd = np.random.random(len(theta))
w, = np.where(rnd < weights)
if w.size > 0:
if w.size > nleft:
theta = theta[w][:nleft]
phi = phi[w][:nleft]
else:
theta = theta[w]
phi = phi[w]
pos[:,ndone:ndone+min(w.size,nleft)] = np.array([theta,phi])
ndone += w.size
nleft -= w.size
return pos
wiener = S.dot(SNi)
print "Applying Wiener filter...",
sys.stdout.flush()
w_solution = wiener.dot(raw_solution)
#plt.imshow(np.log10(wiener))
#plt.colorbar()
#plt.show()
###adding back point source for plotting:
if ps:
for i in range(len(point_sources)):
ra, dec = point_sources[i]
theta = np.pi / 2 - dec
phi = ra
raw_solution[hpf.ang2pix(
nside, theta, phi)] += raw_solution[12 * nside**2 + i]
w_solution[hpf.ang2pix(
nside, theta, phi)] += w_solution[12 * nside**2 + i]
#plt.plot(np.abs(ev))
#plt.show()
hpv.mollview(equatorial_GSM_standard,
min=0,
max=5000,
title='Original map')
hpv.mollview(wiener.dot(equatorial_GSM),
min=0,
max=5000,
title='Wienered map')
#if nside != nside_standard:
#hpv.mollview(equatorial_GSM, min=0,max=5000,fig=2,title='mean map')
wiener = S.dot(SNi)
print "Applying Wiener filter...",
sys.stdout.flush()
w_solution = wiener.dot(solution)
#plt.imshow(np.log10(wiener))
#plt.colorbar()
#plt.show()
###adding back point source:
if ps:
for i in range(len(point_sources)):
ra, dec = point_sources[i]
theta = np.pi/2 - dec
phi = ra
solution[hpf.ang2pix(nside, theta, phi)] += raw_solution[12*nside**2 + i]
w_solution[hpf.ang2pix(nside, theta, phi)] += raw_solution[12*nside**2 + i]
#plt.plot(np.abs(ev))
#plt.show()
#hpv.mollview(equatorial_GSM_standard, min=0,max=5000,fig=1,title='Original map')
#if nside != nside_standard:
#hpv.mollview(equatorial_GSM, min=0,max=5000,fig=2,title='mean map')
hpv.mollview(solution, min=0,max=5000,title='Raw solution, rcond = %i, noise = %i'%(np.log10(rcondA), np.log10(noise)))
hpv.mollview(np.log10(np.abs(solution-equatorial_GSM)/equatorial_GSM), return_projected_map=True, min=-2, max=0,title='log10(relative error), rcond = %i, noise = %i'%(np.log10(rcondA), np.log10(noise)))
hpv.mollview(w_solution, min=0,max=5000,title='Wiener solution, rcond = %i, noise = %i'%(np.log10(rcondA), np.log10(noise)))
hpv.mollview(np.log10(np.abs(w_solution-equatorial_GSM)/equatorial_GSM), return_projected_map=True, min=-2, max=0,title='log10(relative error wiener), rcond = %i, noise = %i'%(np.log10(rcondA), np.log10(noise)))
#hpv.mollview(solution/B.dot(equatorial_GSM_standard), min=0, max=3, fig=4,title='sol/mean')
print " "
def hpix(self, nside=64, nest=True):
return ang2pix(nside, -self.dec.rad + pi / 2, self.ra.rad, nest)
def test(temp, file_path):
import time
import numpy as np
import pandas as pd
import healpy.pixelfunc as pf
####################
# DF definition
####################
filter_set = ['u', 'g', 'r', 'i', 'z']
filter_set = [f'{fs}_ap'
for fs in filter_set] + [f'{fs}_ab' for fs in filter_set]
zrange = [[0.0, 0.3], [0.3, 0.6], [0.6, 0.9], [0.9, 1.2], [1.2, 2.5]]
####################
# Create galaxies
####################
temp = np.linspace(22.8, 24.8, 1000)
temp = {f: temp for f in filter_set}
temp['id_galaxy'] = np.arange(1000)
####################
# Position galaxies
####################
'''A ring order is used for this set.'''
pix_region = pf.get_all_neighbours(32,
theta=90,
phi=0,
nest=False,
lonlat=True)
pix_region = np.concatenate([
pix_region,
np.array([pf.ang2pix(32, 90, 0, nest=False, lonlat=True)])
])
pix_map = np.zeros(pf.nside2npix(32))
pix_map[pix_region] = 1
pix_map = pf.ud_grade(pix_map, 64, order_in='RING', order_out='RING')
pix_region = np.arange(pf.nside2npix(64))[pix_map > 0]
ra_region, dec_region = pf.pix2ang(64, pix_region, nest=False, lonlat=True)
data = []
for i in range(len(pix_region)):
for j in range(len(zrange)):
temp['ra'] = ra_region[i]
temp['dec'] = dec_region[i]
temp['zobs'] = (zrange[j][0] + zrange[j][1]) / 2
data.append(pd.DataFrame(temp))
data = pd.concat(data)
big_pixels = pf.ang2pix(32,
data['ra'],
data['dec'],
nest=False,
lonlat=True)
big_pixel_ID = np.unique(big_pixels)
####################
# File save
####################
fnames = []
for pix in big_pixel_ID:
fname = f'{file_path}galaxies_test_{pix}.csv'
fnames.append(fname)
data_subset = data.loc[big_pixels == pix]
data_subset.to_csv(fname, index=False)
print('Test data created')
return (fnames)
def model(self, TP_HTG_KING, HTG2_LIVETIME, NHPSIDE=256, THRESHOLD_ANGDIST=15, tpl_path_perf=('/nfs/farm/g/glast/u/mtakahas/v20r09p09_G1haB1/S18/S18V200909_020RAWE20ZDIR020ZCS000wwoTRKwoMCZDIR00woRWcatTwo_15/S18ZDIR020catTwoZDIR060_E28binx_Cth40bins_CalOnly_R100_perf.root', '/nfs/farm/g/glast/u/mtakahas/v20r09p09_G1haB1/S18/S18V200909_020RAWE20ZDIR020ZCS000wwoTRKwoMCZDIR00woRWcatTwo_15/S18ZDIR020catTwoZDIR060_E28binx_Cth40bins_CalOnly_R30_perf.root', '/nfs/farm/g/glast/u/mtakahas/v20r09p09_G1haB1/S18/S18V200909_020RAWE20ZDIR020ZCS000wwoTRKwoMCZDIR00woRWcatTwo_15/S18ZDIR020catTwoZDIR060_E28binx_Cth40bins_CalOnly_R10_perf.root'), aeff_regular=None):
"""Model the point source with the PSF of King function. Energy dispersion is ignored currently.
"""
PIX_TRUE = hppf.ang2pix(NHPSIDE, pi/2.-self.DEC_RAD, self.RA_RAD) # #Pixel the true position of the source locates in
TP_DIR_TRUE = (pi/2.-self.DEC_RAD, self.RA_RAD)
print 'NSIDE:', NHPSIDE
NPIX = hppf.nside2npix(NHPSIDE)
print NPIX, 'pixels.'
print 'Pixel resolution:', degrees(hppf.nside2resol(NHPSIDE)), 'deg'
sa_pix = hppf.nside2pixarea(NHPSIDE) # Solid angle of a pixel [sr]
print 'Solid angle of each pixel:', sa_pix, 'sr'
coo_src = SkyCoord(self.RA, self.DEC, unit="deg") # True coordinate of the source
THRESHOLD_ANGDIST_RADIANS = radians(THRESHOLD_ANGDIST)
self.dctt_htg_sed_model_enr[NHPSIDE] = {}
htg_aeff_regular = None
if aeff_regular!=None:
file_aeff_regular = ROOT.TFile(aeff_regular, 'READ')
htg_aeff_regular = file_aeff_regular.Get('htgEaRegular')
print htg_aeff_regular.GetName()
# ROI
set_pix_roi = set([])
dict_angdist = {}
for ipix in range(NPIX):
tp_pix = hppf.pix2ang(NHPSIDE, ipix)
dict_angdist[ipix] = hp.rotator.angdist(tp_pix, TP_DIR_TRUE) # Angular distance between the source and each pixel in radians
if dict_angdist[ipix] < THRESHOLD_ANGDIST_RADIANS:
set_pix_roi.add(ipix)
print 'ROI pixels:', set_pix_roi
HTG1_LT = HTG2_LIVETIME.ProjectionX("{0}_projTheta".format(HTG2_LIVETIME.GetName()), 1, HTG2_LIVETIME.GetYaxis().FindBin(self.ZENITH_CUT)-1)
# Preparation of ON region setup
dct_path_perf = {}
dct_file_perf = {}
dct_htg_psf95 = {}
dct_htg_acc = {}
# PSF
fc_King_annulus = ROOT.TF1("fc_King_annulus", "TMath::Sin(x)*[0]*(1.-1./[2])*pow(1.+(x/[1])**2/2./[2],-[2])/[1]**2", 0, pi)
fc_King = ROOT.TF1("fc_King", "[0]*(1.-1./[2])*pow(1.+(x/[1])**2/2./[2],-[2])/2./TMath::Pi()/[1]**2", 0, pi)
for (icla,cla) in enumerate(self.TPL_STR_CLASS):
print '======================='
print cla
print '======================='
dct_path_perf[cla] = tpl_path_perf[icla]
dct_file_perf[cla] = ROOT.TFile(dct_path_perf[cla], 'READ')
dct_htg_psf95[cla] = dct_file_perf[cla].Get('psf_q95_hist')
dct_htg_acc[cla] = dct_file_perf[cla].Get('acc_cth_hist')
htg2_model = ROOT.TH2D("htg2_model_{0}_NSIDE{1}".format(cla, NHPSIDE), "Model of {0} {1} (NSIDE={2})".format(cla, self.NAME, NHPSIDE), NPIX, 0, NPIX, self.NBIN_ENERGY, self.EDGE_ENERGY_LOW, self.EDGE_ENERGY_UP)
htg2_model_on = htg2_model.Clone('{0}_ON'.format(htg2_model.GetName()))
htg_sed_model = ROOT.TH2D('htg_sed_model_{0}'.format(cla), 'SED model of {0} {1}'.format(cla, self.NAME), self.HTG_SED.GetNbinsX(), self.HTG_SED.GetXaxis().GetXmin(), self.HTG_SED.GetXaxis().GetXmax(), dct_htg_acc[cla].GetNbinsY(), dct_htg_acc[cla].GetYaxis().GetXmin(), dct_htg_acc[cla].GetYaxis().GetXmax())
for iEne in range(1, self.HTG_SED.GetNbinsX()+1):
print 'Energy:', self.HTG_SED.GetXaxis().GetBinLowEdge(iEne), '-', self.HTG_SED.GetXaxis().GetBinUpEdge(iEne)
flux_true_itgl = self.HTG_SED.GetBinContent(iEne)
fluxerr_true_itgl = self.HTG_SED.GetBinError(iEne)
print ' Flux:', flux_true_itgl, '+/-', fluxerr_true_itgl, '[photons cm^-2 s^-1]'
for iTh in range(1, HTG1_LT.GetNbinsX()+1):
scale_aeff = dct_htg_acc[cla].GetBinContent(dct_htg_acc[cla].GetXaxis().FindBin(self.HTG_SED.GetXaxis().GetBinCenter(iEne)), dct_htg_acc[cla].GetYaxis().FindBin(HTG1_LT.GetXaxis().GetBinCenter(iTh))) / (2*math.pi*dct_htg_acc[cla].GetYaxis().GetBinWidth(dct_htg_acc[cla].GetYaxis().FindBin(HTG1_LT.GetXaxis().GetBinCenter(iTh)))) # Effective area [m^2]
if htg_aeff_regular!=None:
#print '=-=-=-=-='
#print scale_aeff
scale_aeff = scale_aeff + htg_aeff_regular.GetBinContent(htg_aeff_regular.GetXaxis().FindBin(10**self.HTG_SED.GetXaxis().GetBinCenter(iEne)), htg_aeff_regular.GetYaxis().FindBin(HTG1_LT.GetXaxis().GetBinCenter(iTh)))
#print scale_aeff
scale_exp = scale_aeff * 100.**2 * HTG1_LT.GetBinContent(iTh) # Integrated exposure value for a certain energy and inclination angle [cm^2 s]
nphoton = flux_true_itgl*scale_exp
nphotonerr = fluxerr_true_itgl*scale_exp
htg_sed_model.Fill(htg_sed_model.GetXaxis().GetBinCenter(iEne), htg_sed_model.GetYaxis().GetBinCenter(iTh), nphoton)
kxbin = TP_HTG_KING[0].GetXaxis().FindBin(self.HTG_SED.GetXaxis().GetBinCenter(iEne))
kybin = TP_HTG_KING[0].GetYaxis().FindBin(HTG1_LT.GetBinCenter(iTh))
if nphoton>0 and kxbin>0 and kybin>0:
print ' cos(Inclination angle):', HTG1_LT.GetXaxis().GetBinLowEdge(iTh), '-', HTG1_LT.GetXaxis().GetBinUpEdge(iTh)
print ' Effective area:', scale_aeff, 'm^2'
print ' Exposure:', scale_exp, 'cm^2 s'
print ' Number of photons:', nphoton, '+/-', nphotonerr
print ''
for ipar in range(3): # Setting the parameters of King function
# PSF
par_value = TP_HTG_KING[ipar].GetBinContent(kxbin, kybin)
#print ' Parameter No.{0}:'.format(ipar), par_value
fc_King_annulus.FixParameter(ipar, par_value)
fc_King.FixParameter(ipar, par_value)
factor_norm = 1.0/fc_King_annulus.Integral(0, pi)
#print ' Normalization factor:', factor_norm
for ipix in set_pix_roi:
scale_psf = fc_King.Eval(dict_angdist[ipix])
htg2_model.Fill(ipix+0.5, htg2_model.GetYaxis().GetBinCenter(iEne), nphoton*scale_psf*factor_norm*sa_pix)
# for cla in self.TPL_STR_CLASS:
deg_psf95 = dct_htg_psf95[cla].GetBinContent(dct_htg_psf95[cla].FindBin(self.HTG_SED.GetBinCenter(iEne)))
if radians(min(15, deg_psf95))>dict_angdist[ipix]:
htg2_model_on.SetBinContent(ipix, iEne, 1)
else:
htg2_model_on.SetBinContent(ipix, iEne, 0)
if self.HTG_SED.GetBinContent(iEne)!=0:
htg_sed_model.SetBinError(iEne, iTh, self.HTG_SED.GetBinError(iEne)/self.HTG_SED.GetBinContent(iEne)*htg_sed_model.GetBinContent(iEne, iTh))
else:
htg_sed_model.SetBinError(iEne, iTh, 0)
print htg2_model_on.Integral()
htg2_model_on.Multiply(htg2_model)
for ienr in range(1, htg2_model_on.GetYaxis().GetNbins()+1):
for ipix in range(1, htg2_model_on.GetXaxis().GetNbins()+1):
content = htg2_model.GetBinContent(ipix, ienr)
content_err = htg2_model.GetBinError(ipix, ienr)
content_on = htg2_model_on.GetBinContent(ipix, ienr)
if content>0:
htg2_model_on.SetBinError(ipix, ienr, content_err*content_on/content)
else:
htg2_model_on.SetBinError(ipix, ienr, 0)
print htg2_model_on.Integral()
htg_sed_model_on = htg2_model_on.ProjectionY('{0}_ON'.format(htg_sed_model.GetName()))
print 'ON photon number:', htg_sed_model_on.Integral()
htg_sed_model_enr = htg_sed_model.ProjectionX('{0}_projEnergy'.format(htg_sed_model.GetName()))
for ienr in range(1, htg_sed_model_on.GetXaxis().GetNbins()+1):
content = htg_sed_model_enr.GetBinContent(ienr)
content_err = htg_sed_model_enr.GetBinError(ienr)
content_on = htg_sed_model_on.GetBinContent(ienr)
if content>0:
htg_sed_model_on.SetBinError(ienr, content_err*content_on/content)
else:
htg_sed_model_on.SetBinError(ienr, 0)
print 'Making map...'
arr_map = []
arr_map_energy = []
for iEne in range(0, self.HTG_SED.GetNbinsX()+1):
arr_map.append([])
if iEne>0:
arr_map_energy.append((self.HTG_SED.GetXaxis().GetBinLowEdge(iEne), self.HTG_SED.GetXaxis().GetBinUpEdge(iEne)))
for ipix in range(htg2_model.GetXaxis().GetNbins()):
if ipix in set_pix_roi:
arr_map[-1].append(htg2_model.GetBinContent(ipix+1, iEne))
else:
arr_map[-1].append(hppf.UNSEEN)
else:
arr_map_energy.append((self.HTG_SED.GetXaxis().GetBinLowEdge(1), self.HTG_SED.GetXaxis().GetBinUpEdge(self.HTG_SED.GetNbinsX())))
for ipix in range(htg2_model.GetXaxis().GetNbins()):
if ipix in set_pix_roi:
arr_map[-1].append(htg2_model.Integral(ipix+1, ipix+1, 1, htg2_model.GetNbinsY()))
else:
arr_map[-1].append(hppf.UNSEEN)
print ' Energy:', arr_map_energy[-1][0], '-', arr_map_energy[-1][1]
nparr_map = np.array(arr_map[-1]) # Take this energy bin
hp.visufunc.cartview(nparr_map, iEne, self.tp_rotate, unit='cm^-2 s^-1 / {0:.1E} sr'.format(sa_pix), lonra=[-THRESHOLD_ANGDIST, THRESHOLD_ANGDIST], latra=[-THRESHOLD_ANGDIST, THRESHOLD_ANGDIST], title='{0} ({1:.3f} - {2:.3f} GeV)'.format(self.NAME, pow(10, arr_map_energy[-1][0]-3), pow(10, arr_map_energy[-1][1]-3)), min=0, flip='astro')
plt.savefig("png/Model-{0}_{1}_NSIDE{2}_{3}-{4}.png".format(self.NAME, cla, NHPSIDE, int(100*arr_map_energy[-1][0]+0.5), int(100*arr_map_energy[-1][1]+0.5)))
plt.close()
self.FILE_OUT.cd()
htg_sed_model.Write()
htg_sed_model_enr.Write()
htg2_model.Write()
htg_sed_model_on.Write()
self.dctt_htg_sed_model_enr[NHPSIDE][cla] = copy.copy(htg_sed_model_enr)
self.HTG_SED.Write()
def main():
global isNearest
global maxgalid
global db_flag
global atc_flag
if args.verbose > 0:
print 'verbose = ', args.verbose
if args.testdb > 0:
sngals_file = 'SNGALS_TEST'
sngals_file_id = 'COADD_OBJECTS_ID' #'OBJECTS_ID'
else:
sngals_file = 'SNGALS'
sngals_file_id = 'OBJECTS_ID' # 'COADD_OBJECTS_ID'
start_time = time.time()
hostname = 'leovip148.ncsa.uiuc.edu'
port = 1521
dbname = args.dbname
username = args.username
password = args.password
dsn = cx_Oracle.makedsn(hostname, port, service_name=dbname)
connection = cx_Oracle.Connection(username, password, dsn)
cursor = connection.cursor()
#----------------------------------------------------------------------------------------------
#----------- Query all transients that have no entry in the SNGALS table. ---------------------
# ---------- These are the galaxies we are to host-match --------------------------------------
#----------------------------------------------------------------------------------------------
query = (
"SELECT c.transient_name, c.SNID, c.ra, c.dec, "
#"substr(c.transient_name,6,2), FIELD "
"from SNCAND c LEFT JOIN SNGALS g on c.SNID=g.SNID "
"where c.transient_name is not NULL "
"and c.transient_status >= 0 and c.snfake_id = 0 "
"and g.SNGALID is NULL ")
query_new = (
"SELECT c.transient_name, c.SNID, c.ra, c.dec "
#"c.FIELD as field "
"from SNCAND c LEFT JOIN " + sngals_file + " g on c.SNID=g.SNID "
#TO BE PUT BACK LATER! "where c.numepochs_ml>=1 and c.numobs_ml>=2 "
"where (c.transient_status is NULL or c.transient_status >=0) "
"and c.snfake_id=0 and c.cand_type >=0 "
"and g.SNGALID is NULL "
#"and g.SNGALID is NULL and c.FIELD is not NULL ")
"and c.season=" + str(args.season) + " ")
#print query_new
cursor.execute(query_new)
data = cursor.fetchall()
if args.verbose > 0:
print "Number of DES Candidates to match = ", len(data)
#Sort by field to lessen memory issues on loading large files
if len(data) > 0:
#data.sort(key=lambda x:x[4])
data = np.array(data)
else:
print
print "WARNING:"
print "The list of galaxies to host match is empty."
print "The query below has failed:"
print
print query_new
print
print "This could be due to a previous successful run."
print "If you wish to rerun, please clear the database using the following command:"
print
print "delete from SNGALS where season=" + str(args.season) + ';'
print
return
#----------------------------------------------------------------------------------------------
#------------ Query the maximum SNGALID to start counting with --------------------------------
#----------------------------------------------------------------------------------------------
query = "SELECT max(SNGALID) from " + sngals_file
cursor.execute(query)
galmax = cursor.fetchall()
if galmax[0][0] is None:
maxgalid = 0
else:
maxgalid = galmax[0][0]
if args.verbose > 0:
print "Max GALID = ", maxgalid
#----------------------------------------------------------------------------------------------
#------------- Select all field names where a transient is ------------------------------------
#------------- lacking an identified host and requires matching -------------------------------
#----------------------------------------------------------------------------------------------
ra_cand = np.array(data[:, 2], dtype=float)
dec_cand = np.array(data[:, 3], dtype=float)
idx = [ra_cand < 0.0]
ra_cand[idx] = ra_cand[idx] + 360.
ra_cand = ra_cand * pi / 180.
dec_cand = (90. - dec_cand) * pi / 180.
pix = ang2pix(32, dec_cand, ra_cand)
fields_cands = np.array(pix)
fields = np.unique(pix)
#----------------------------------------------------------------------------------------------
#------------- Select all the host galaxies that have already been assigned -------------------
#------------- to a transient; this way we reassign the same SNGALID if one -------------------
#------------- has previously been used -------------------------------------------------------
#----------------------------------------------------------------------------------------------
query = ("select SNID, " + sngals_file_id + " , SNGALID from " +
sngals_file + " where " + sngals_file_id + " is not NULL")
cursor.execute(query)
temp = cursor.fetchall()
temp = np.array(temp)
current_host = np.zeros(np.sum(max(len(temp),1)), \
dtype={'names':['SNID','SNGALID','COADD_ID'], \
'formats':['S8','S8','S12']})
if args.verbose > 0:
print 'Number of objects currently in SNGALS: ', len(temp)
if len(temp):
current_host['SNID'] = temp[:, 0]
current_host['COADD_ID'] = temp[:, 1]
current_host['SNGALID'] = temp[:, 2]
else:
current_host['SNID'] = -1
current_host['COADD_ID'] = -1
current_host['SNGALID'] = -1
#----------------------------------------------------------------------------------------------
#------------- Loop through the data, querying for nearby galaxies ----------------------------
#------------- and writing results to the database --------------------------------------------
#----------------------------------------------------------------------------------------------
for thisField in fields:
if args.verbose > 0:
print "Starting field = ", thisField
# Find all candidates in this field ...
NCands = 0
if len(data) > 0:
index = np.where(pix == thisField)
index = [s for s in data[index]]
NCands = len(index)
pix_edges = get_all_neighbours(32, thisField)
if args.verbose > 0:
print "\t", NCands, " entries to match in this field"
if NCands > 0:
# Read in field catalog using 'loadtxt' and store relevant data to an array
if thisField < 10000:
filename = INPUT_DIR + "GW_cat_hpx_0" + str(
thisField) + ".fits"
else:
filename = INPUT_DIR + "GW_cat_hpx_" + str(thisField) + ".fits"
if args.verbose > 0:
print "\tReading in catalog from file . . . . \n"
print filename + '\n'
h = fits.open(filename)
cat = h[1].data
for pix_edge in pix_edges:
if pix_edge < 10000:
filename = INPUT_DIR + "GW_cat_hpx_0" + str(
pix_edge) + ".fits"
else:
filename = INPUT_DIR + "GW_cat_hpx_" + str(
pix_edge) + ".fits"
h = fits.open(filename)
cat_edge = h[1].data
cat = np.concatenate((cat, cat_edge))
# cat = np.loadtxt(filename, \
# dtype={'names': ('COADD_OBJECTS_ID','RA','DEC','PHOTOZ','PHOTOZ_ERR', \
# 'MAG_AUTO_G','MAG_AUTO_R','MAG_AUTO_I','MAG_AUTO_Z', \
# 'MAGERR_AUTO_G','MAGERR_AUTO_R','MAGERR_AUTO_I', \
# 'MAGERR_AUTO_Z','MAG_APER_4_G','MAG_APER_4_R', \
# 'MAG_APER_4_I','MAG_APER_4_Z','MAGERR_APER_4_G', \
# 'MAGERR_APER_4_R','MAGERR_APER_4_I','MAGERR_APER_4_Z', \
# 'MAG_DETMODEL_G','MAG_DETMODEL_R','MAG_DETMODEL_I', \
# 'MAG_DETMODEL_Z','MAGERR_DETMODEL_G', \
# 'MAGERR_DETMODEL_R','MAGERR_DETMODEL_I',\
# 'MAGERR_DETMODEL_Z','THETA_IMAGE','A_IMAGE','B_IMAGE', \
# 'GALFLAG','CATALOG'), 'formats': \
# ('S12','f8','f8','f8','f8','S10','S10','S10','S10','S10','S10', \
# 'S10','S10','S10','S10','S10','S10','S10','S10','S10','S10', \
# 'S10','S10','S10','S10','S10','S10','S10','S10',\
# 'f8','f8','f8','f4','S12')}, \
# skiprows=1, usecols=[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,\
# 20,21,22,23,24,25,26,27,28,29,30,31,32,33])
if args.verbose > 0:
print "\tCatalog contains", cat.shape[0], "rows\n"
#Begin Loop...
for i, entry in enumerate(index): #this will not work
transient_name = entry[0]
if transient_name is None:
transient_name = 'NULL'
SNID = entry[1]
ra = float(entry[2])
dec = float(entry[3])
#field = entry[4]
if args.verbose > 0:
print '\ni=', i, entry[0], entry[
1], '(RA, dec) = (', ra, ',', dec, ')'
# First pass search over a box
jndex = (cat['RA'] > ra - farc/math.cos(dec*rad) ) & \
(cat['RA'] < ra + farc/math.cos(dec*rad) ) & \
(cat['DEC'] > dec - farc) & (cat['DEC'] < dec + farc)
# store all catalog objects within tolerance in a temporary array
array = cat[jndex]
#Keep track of whether a host has been assigned for this object.
#If not, mark as hostless in SNGALS to prevent repeated searches
#for this candidate on subsequent runs.
hostlessFlag = 1
if len(array) > 0:
# distance to SN in arcsec
dist = 3600. * np.sqrt( ( np.cos( dec*rad ) * ( array['RA'] - ra ) )**2 + \
( array['DEC'] - dec )**2 )
# Check if galaxies are within search radius
circle = dist < search
if np.sum(circle) > 0:
# define structured array containing potential host info
hostinfo = np.zeros(np.sum(circle), \
dtype={'names':['transient_name','SNID','SNGALID', \
'COADD_ID','RA','DEC','sep','DLR', \
'rank','host', 'X2','Y2','XY','A','B', \
'rPHI','A_IMAGE','B_IMAGE','THETA_IMAGE'],\
'formats':['S12','S8','S8','S20','f8','f8','f8', \
'f8','i4','i4','f8','f8','f8','f8','f8',\
'f8','f8','f8','f8']})
hostinfo['transient_name'][:] = transient_name
hostinfo['SNID'][:] = SNID
hostinfo['sep'] = dist[circle] # SN-galaxy separation
hostinfo['COADD_ID'] = array['ID'][circle]
hostinfo['RA'] = array['RA'][circle]
hostinfo['DEC'] = array['DEC'][circle]
hostinfo['A_IMAGE'] = array['A'][circle]
hostinfo['B_IMAGE'] = array['B'][circle]
hostinfo['THETA_IMAGE'] = array['THETA'][circle]
#print array.shape,hostinfo['COADD_ID']
#Deprecated until these parameters are included in DESDM catalogs
#hostinfo['X2'] = array['X2WIN_IMAGE'][circle]
#hostinfo['Y2'] = array['Y2WIN_IMAGE'][circle]
#hostinfo['XY'] = array['XYWIN_IMAGE'][circle]
# Compute d_DLR for all potential hosts
retval = hma.get_DLR_ABT( ra, dec, hostinfo['RA'], hostinfo['DEC'],\
hostinfo['A_IMAGE'], hostinfo['B_IMAGE'], \
hostinfo['THETA_IMAGE'], hostinfo['sep'] )
hostinfo['DLR'] = retval[0]
hostinfo['A'] = retval[1]
hostinfo['B'] = retval[2]
hostinfo['rPHI'] = retval[3]
# Compute HOST_CONFUSION parameter and insert into SNCAND # RRG
#HC = hma.compute_HC(hostinfo['DLR'])
#if args.verbose > 0:
# print "\tHOST_CONFUSION = ", HC
#query_HC = ("UPDATE SNCAND SET HOST_CONFUSION = {0} "
# "WHERE SNID={1}").format(HC, SNID)
#if not args.test:
# cursor.execute(query_HC)
# connection.commit()
# continue # RRG -- skip the host matching part
#------------------------------------------------------------------------------
#-------------- Order by Angular Separation -----------------------------------
#-------------- Results are output to file, and compared to -------------------
#---------------DLR, but are not written to database --------------------------
#------------------------------------------------------------------------------
hostinfo.sort(order='sep')
for k in range(0, len(hostinfo)):
# rank ordered galaxies
hostinfo['rank'][k] = k + 1
s = (
"{:10s} {:7s} {:30s} {:10.5f} {:10.5f} {:10.5f} "
"{:10.5f} {:8.4f} {:8.4f} {:8.4f} {:d}\n"
)
s = s.format(hostinfo['transient_name'][k], hostinfo['SNID'][k], \
hostinfo['COADD_ID'][k], hostinfo['RA'][k], \
hostinfo['DEC'][k], hostinfo['sep'][k], hostinfo['DLR'][k], \
hostinfo['A'][k], hostinfo['B'][k], \
hostinfo['rPHI'][k], hostinfo['rank'][k])
if args.verbose > 1:
print '\tSep: ',k, hostinfo['sep'][k], hostinfo['DLR'][k], \
hostinfo['rank'][k], hostinfo['X2'][k], hostinfo['Y2'][k], \
hostinfo['XY'][k]
file_sep.write(s)
hostinfo.sort(order='DLR')
if hostinfo['DLR'][0] == 99.99:
hostinfo.sort(order='sep')
# if nearest ordered DLR host has DLR = 99.99,
# default to nearest separation
# Write to file_mismatch if host by angular separation and DLR are different
# Write both galaxies
if hostinfo['rank'][
0] == 1: # smallest DLR is already nearest galaxy
isNearest += 1 # increment counter for this case
else:
# write smallest angular separation object
r = np.where(hostinfo['rank'] == 1)
s = (
"{:10s} {:7s} {:30s} {:10.5f} {:10.5f} {:10.5f} {:10.5f} "
"{:8.4f} {:8.4f} {:8.4f} {:d}\n")
s = s.format( hostinfo['transient_name'][r[0][0]], \
hostinfo['SNID'][r[0][0]], hostinfo['COADD_ID'][r[0][0]], \
hostinfo['RA'][r[0][0]], hostinfo['DEC'][r[0][0]], \
hostinfo['sep'][r[0][0]], hostinfo['DLR'][r[0][0]], \
hostinfo['A'][r[0][0]], hostinfo['B'][r[0][0]], \
hostinfo['rPHI'][r[0][0]], hostinfo['rank'][r[0][0]] )
file_mismatch.write(s)
hostinfo[
'rank'][:] = 0 # reset all to 0 to clear the nearest galaxy
hostinfo['rank'][
0] = 1 # set smallest DLR to have host=1
# write smallest DLR object
r = np.where(hostinfo['rank'] == 1)
s = (
"{:10s} {:7s} {:30s} {:10.5f} {:10.5f} {:10.5f} {:10.5f} "
"{:8.4f} {:8.4f} {:8.4f} {:d}\n")
s = s.format(hostinfo['transient_name'][r[0][0]], \
hostinfo['SNID'][r[0][0]], hostinfo['COADD_ID'][r[0][0]], \
hostinfo['RA'][r[0][0]], hostinfo['DEC'][r[0][0]], \
hostinfo['sep'][r[0][0]], hostinfo['DLR'][r[0][0]], \
hostinfo['A'][r[0][0]], hostinfo['B'][r[0][0]], \
hostinfo['rPHI'][r[0][0]], hostinfo['rank'][r[0][0]] )
file_mismatch.write(s)
#------------------------------------------------------------------------------
#------------------ Now go through each galaxy by DLR -------------------------
#------------------ (already sorted above) and write to file, -----------------
#------------------ write to DB, ngest into ATC, and tag there ----------------
#------------------------------------------------------------------------------
for k in range(0, len(hostinfo)):
# rank ordered galaxies
if hostinfo['DLR'][k] < DLR_cut:
hostinfo['rank'][k] = k + 1
else:
# assign negative rank if fails DLR cut
hostinfo['rank'][k] = (k + 1) * -1
if hostinfo['rank'][k] == 1:
# assign 'host'=1 only if 'rank'=1, else 'host'=0
hostinfo['host'][k] = 1
else:
hostinfo['host'][k] = 0
s = (
"{:10s} {:7s} {:30s} {:10.5f} {:10.5f} {:10.5f} {:10.5f} "
"{:8.4f} {:8.4f} {:8.4f} {:3d} {:d}\n"
)
s = s.format(hostinfo['transient_name'][k], hostinfo['SNID'][k], \
hostinfo['COADD_ID'][k], hostinfo['RA'][k], \
hostinfo['DEC'][k], hostinfo['sep'][k], hostinfo['DLR'][k], \
hostinfo['A'][k], hostinfo['B'][k], hostinfo['rPHI'][k], \
hostinfo['rank'][k], hostinfo['host'][k] )
if args.verbose > 1:
print '\tDLR: ',k, hostinfo['sep'][k], hostinfo['DLR'][k], \
hostinfo['host'][k],hostinfo['X2'][k], hostinfo['Y2'][k], \
hostinfo['XY'][k]
# Only write top 3 ranked galaxies to file/DB
if k < 3:
# ---------------- Write to File --------------------------------------
file_dlr.write(s)
# ---------------- Write to Database ----------------------------------
temp_galid = 0
db_status = 0
# Does this galaxy already exist in the DB?
kndex = np.where(current_host['COADD_ID'] ==
hostinfo['COADD_ID'][k])
if len(kndex[0]) > 0:
temp_galid = current_host['SNGALID'][
kndex[0][0]]
else:
maxgalid += 1
temp_galid = maxgalid
#print array
kndex = np.where(array['ID'][:] == int(
hostinfo['COADD_ID'][k]))
tt = kndex[0][0]
#hostname = hostinfo['SURVEY'][k] + "_ID-" + str(array['ID'][tt])
query = (
"INSERT INTO " + sngals_file + " ( " +
sngals_file_id + ", RA, DEC, PHOTOZ, "
"PHOTOZ_ERR, MAG_AUTO_I, "
"THETA_IMAGE, A_IMAGE, B_IMAGE, "
"SNID, TRANSIENT_NAME, SNGALID, SEPARATION, "
"DLR_RANK, HOST, SEASON, SURVEY,MAG_APER_4_I ) " #Add DLR again once the a b issues are fixed
"VALUES ("
"{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},"
"{13},{14},{15},{16},{17} )")
# The Galaxy Catalogs use 99.999 for mag uncertainties.
# We have an upper limit of 9.999.
# Here's a very stupid fix to that.
#if float(array['MAGERR_AUTO_G'][tt]) >= 10:
# array['MAGERR_AUTO_G'][tt] = 9.000
#if float(array['MAGERR_AUTO_R'][tt]) >= 10:
# array['MAGERR_AUTO_R'][tt] = 9.000
#if float(array['MAGERR_AUTO_I'][tt]) >= 10:
# array['MAGERR_AUTO_I'][tt] = 9.000
#if float(array['MAGERR_AUTO_Z'][tt]) >= 10:
# array['MAGERR_AUTO_Z'][tt] = 9.000
#if float(array['MAGERR_APER_4_G'][tt]) >= 10:
# array['MAGERR_APER_4_G'][tt] = 9.000
#if float(array['MAGERR_APER_4_R'][tt]) >= 10:
# array['MAGERR_APER_4_R'][tt] = 9.000
#if float(array['MAGERR_APER_4_I'][tt]) >= 10:
# array['MAGERR_APER_4_I'][tt] = 9.000
#if float(array['MAGERR_APER_4_Z'][tt]) >= 10:
# array['MAGERR_APER_4_Z'][tt] = 9.000
#if float(array['MAGERR_DETMODEL_G'][tt]) >= 10:
# array['MAGERR_DETMODEL_G'][tt] = 9.000
#if float(array['MAGERR_DETMODEL_R'][tt]) >= 10:
# array['MAGERR_DETMODEL_R'][tt] = 9.000
#if float(array['MAGERR_DETMODEL_I'][tt]) >= 10:
# array['MAGERR_DETMODEL_I'][tt] = 9.000
#if float(array['MAGERR_DETMODEL_Z'][tt]) >= 10:
# array['MAGERR_DETMODEL_Z'][tt] = 9.000
# Now execute query
query = query.format(array['ID'][tt],array['RA'][tt],\
array['DEC'][tt],array['ZPHOTO'][tt],\
array['ZPHOTO_ERR'][tt],array['MAG_I'][tt],\
array['THETA'][tt],array['A'][tt],\
array['B'][tt],\
hostinfo['SNID'][k],\
hostinfo['transient_name'][k],temp_galid,\
hostinfo['sep'][k],\
hostinfo['rank'][k],hostinfo['host'][k],args.season,'\''+array['CATALOG'][tt]+'\'',array['MAG_I'][tt])
if args.verbose > 0:
print hostinfo['SNID'][k],hostinfo['transient_name'][k],\
array['ID'][tt],\
array['THETA'][tt],array['A'][tt],\
array['B'][tt]
dbstring = 'SN: {0:<12}{1:<16}{2:<16}{3:<12}{4:<12}{5:>10}{6:>7}\n'
hostlessFlag = 0
try:
if not args.test:
cursor.execute(query)
connection.commit()
db_status = 0
except:
db_status = 1
#db_flag += 1
#print hostlessFlag
logfile_db.write(
dbstring.format(
hostinfo['SNID'][k],
hostinfo['transient_name'][k],
array['ID'][tt], array['RA'][tt],
array['DEC'][tt], hostinfo['rank'][k],
db_status))
#-------------------- Ingest to ATC (if not star ) --------------------
#if int(array['GALFLAG'][tt]) == 1:
# atcstring = 'SN: {0:<12}{1:<25}{2:<16.8f}{3:<16.8f}{4:<7d}\n'
# atc_status = 0
#Only ingest the host galaxies if a) we set the ingest flag, and
# b) the object the galaxies are associated with have a name.
# While this latter step is not strictly necessary, it saves time
# by avoiding ingesting lots of unneccesary entries.
#if args.atc and transient_name != 'NULL':
# try:
# host_incept = atcfuncs.create_incept_post(hostname,
# array['RA'][tt],
# array['DEC'][tt])
# except:
# atc_status += 1
# try:
# out = atcfuncs.send_posts(host_incept,
# test_mode = args.test)
# if args.verbose > 0:
# print ('Successfully ingested host of ' +
# hostinfo['transient_name'][k] +
# ' as galaxy ' + hostname )
# except:
# # Assumes that all ATC post errors are due to the
# # host galaxy already existing in the DB.
# print ('WARNING: ' + hostinfo['transient_name'][k] +
# ' host ' + hostname + ' already exists in ATC.')
#
# if int(hostinfo['rank'][k]) == 1:
# host_tag = atcfuncs.create_host_post(
# hostinfo['transient_name'][k],hostname
# )
# try:
# out = atcfuncs.send_posts(host_tag,
# test_mode = args.test)
# except:
# print ('WARNING: unable to tag transient ' +
# hostinfo['transient_name'][k] +
# ' with host ' + hostname )
# atc_status += 2
#
#else:
# # If there is no attempt to write to the ATC, then candidates
# # by definition remain to be uploaded
# atc_status += 4
#
#if atc_status > 0 and transient_name != 'NULL':
# atc_flag += 1
#If the atc_status flag is set, we print a failure warning
#UNLESS the SN candidate the host is matched to has no name,
# in which case there is nowhere to upload to.
# Write to logfile for potential later ingestion
#logfile_atc.write( atcstring.format( transient_name,
# hostname, array['RA'][tt],
# array['DEC'][tt], atc_status ) )
#At this point we know whether the hostlessFlag has been updated, or if the SNID
# has no matching host galaxy. If there is none, update with -1 values
if hostlessFlag:
nohost_GALID = -1
nohost_RANK = 0
query = (
"INSERT INTO " + sngals_file +
" (SNID, TRANSIENT_NAME, SNGALID, DLR_RANK) VALUES "
" ({0}, '{1}', {2}, {3}) ")
query = query.format(SNID, transient_name, nohost_GALID,
nohost_RANK)
print query
#if args.verbose > 0:
# print SNID, transient_name,' NO HOST'
dbstring = 'SN: {0:<12}{1:<16}{2:<16}{3:<12}{4:<12}{5:>10}{6:>7}\n'
try:
if not args.test:
cursor.execute(query)
connection.commit()
db_status = 0
except:
db_status = 1
#db_flag += 1
#### I have added this line as it was giving error (didn't know what to put as db_status in logfile_db.write)
if args.test:
db_status = 1
logfile_db.write(
dbstring.format(SNID, transient_name, 'N/A', 'N/A',
'N/A', 0, db_status))
#------------------------------------------------------------------------------
#------------------- Summarize and close log files; Post to ATC ---------------
#------------------------------------------------------------------------------
print 'db_flag ', db_flag
if db_flag == 0:
logfile_db.write('#HostMatch Database Update: SUCCESS\n')
print 'HostMatch Database Update: SUCCESS'
else:
logfile_db.write('#HostMatch Database Update: FAILURE '
'on {} uploads\n'.format(db_flag))
logfile_db.write('#Re-run HostMatch. No input files needed.\n')
print 'HostMatch Database Update: FAILURE '
#if (args.atc and atc_flag == 0):
# logfile_atc.write('#HostMatch ATC Posting & Tagging: SUCCESS\n')
#else:
# if args.atc:
# logfile_atc.write('#HostMatch ATC Posting & Tagging: FAILURE '
# 'on {} posts\n'.format(atc_flag))
# print 'HostMatch ATC Posting & Tagging: FAILURE'
# else:
logfile_atc.write('#HostMatchATC Posting & Tagging: SKIPPED '
'on ALL posts\n')
print 'HostMatch ATC Posting & Tagging: SKIPPED'
logfile_atc.write('#Re-run with --input flag set and input file:\n')
logfile_atc.write('#{0}'.format(logfile_name_atc))
print 'Run with desHostMatchFix with input file set as {0}'.format(
logfile_name_atc)
if args.test:
logfile_atc.write(
'#...Except this was only a TEST, so nothing has REALLY happened!\n'
)
print '...Except this was only a TEST, so nothing has REALLY happened!'
file_dlr.close()
file_sep.close()
file_mismatch.close()
logfile_atc.close()
logfile_db.close()
connection.close()
def a2p(radec):
return ang2pix(nside, radec[0], radec[1], nest=True, lonlat=True)
ubl for ubl in infoubl
if la.norm(ubl) < inclusion_thresh * (nside_target * 299.792458 / freq)
]
print "%i UBLs to include" % len(ubls)
if len(tlist) * len(ubls) < 12 * nside**2:
for i in range(5):
print 'Not enough degree of freedom! %i*%i<%i.' % (
len(tlist), len(ubls), 12 * nside**2)
A = np.empty((len(tlist) * len(ubls), nps), dtype='complex64')
for i in range(nps):
ra, dec = point_sources[i]
theta = np.pi / 2 - dec
phi = ra
theta_heal, phi_heal = hpf.pix2ang(nside, hpf.ang2pix(nside, theta, phi))
A[:, i] = np.array([
vs.calculate_pointsource_visibility(phi_heal,
np.pi / 2. - theta_heal,
d,
freq,
beam_heal_equ=beam_heal_equ,
tlist=tlist) for d in ubls
]).flatten()
print float(time.time() - timer) / 60.
if all_ubl:
A.tofile('/home/omniscope/data/GSM_data/AmatrixPS_allubl_nside%i_%iby%i_' %
(nside, len(A), len(A[0])) + infofile.split('/')[-1])
else:
A.tofile('/home/omniscope/data/GSM_data/AmatrixPS_%iubl_nside%i_%iby%i_' %
vs.initial_zenith = np.array([0, lat_degree*np.pi/180])#self.zenithequ
beam_heal_equ = np.array(sv.rotate_healpixmap(beam_healpix, 0, np.pi/2 - vs.initial_zenith[1], vs.initial_zenith[0]))
timer = time.time()
if all_ubl:
ubls = infoubl
else:
ubls = [ubl for ubl in infoubl if la.norm(ubl) < inclusion_thresh * (nside_target * 299.792458 / freq)]
print "%i UBLs to include"%len(ubls)
if len(tlist)*len(ubls) < 12*nside**2:
for i in range(5):
print 'Not enough degree of freedom! %i*%i<%i.'%(len(tlist), len(ubls), 12*nside**2)
A = np.empty((len(tlist)*len(ubls), nps), dtype='complex64')
for i in range(nps):
ra, dec = point_sources[i]
theta = np.pi/2 - dec
phi = ra
theta_heal, phi_heal = hpf.pix2ang(nside, hpf.ang2pix(nside, theta, phi))
A[:, i] = np.array([vs.calculate_pointsource_visibility(phi_heal, np.pi/2.-theta_heal, d, freq, beam_heal_equ = beam_heal_equ, tlist = tlist) for d in ubls]).flatten()
print float(time.time()-timer)/60.
if all_ubl:
A.tofile('/home/omniscope/data/GSM_data/AmatrixPS_allubl_nside%i_%iby%i_'%(nside, len(A), len(A[0])) + infofile.split('/')[-1])
else:
A.tofile('/home/omniscope/data/GSM_data/AmatrixPS_%iubl_nside%i_%iby%i_'%(len(ubls), nside, len(A), len(A[0])) + infofile.split('/')[-1])
def _catalog_to_cells_neighbor(catalog, radius, order):
"""
Convert a catalog to a list of cells.
This is the original implementation of the `catalog_to_cells`
function which does not make use of the Healpy `query_disc` routine.
Note: this function uses a simple flood-filling approach and is
very slow, especially when used with a large radius for catalog objects
or a high resolution order.
"""
if not isinstance(radius, Quantity):
radius = radius * arcsecond
nside = 2**order
# Ensure catalog is in ICRS coordinates.
catalog = catalog.icrs
# Determine central cell for each catalog entry.
phi = catalog.ra.radian
theta = (pi / 2) - catalog.dec.radian
cells = np.unique(ang2pix(nside, theta, phi, nest=True))
# Iteratively consider the neighbors of cells within our
# catalog regions.
new_cells = cells
rejected = np.array((), dtype=np.int64)
while True:
# Find new valid neighboring cells which we didn't already
# consider.
neighbors = np.unique(
np.ravel(get_all_neighbours(nside, new_cells, nest=True)))
neighbors = np.extract([(x != -1) and (x not in cells) and
(x not in rejected)
for x in neighbors], neighbors)
# Get the coordinates of each of these neighbors and compare them
# to the catalog entries.
(theta, phi) = pix2ang(nside, neighbors, nest=True)
coords = SkyCoord(phi, (pi / 2) - theta, frame='icrs', unit='rad')
(idx, sep2d, dist3d) = coords.match_to_catalog_sky(catalog)
within_range = (sep2d < radius)
# If we didn't find any new cells within range,
# end the iterative process.
if not np.any(within_range):
break
new_cells = neighbors[within_range]
cells = np.concatenate((cells, new_cells))
rejected = np.concatenate(
(rejected, neighbors[np.logical_not(within_range)]))
return cells