def get_Q0(lofar_coords, opt_coords, random_coords, radius):
"""
Compute the Q_0 parameter for a given radius
Parameters:
-----------
lofar_coords : LOFAR coordinates
opt_coords : Optical/NIR coordinates
n_random : Number of random sources to generate
radius : Radius to search for blanks in the random or random blank catalogue
Returns:
--------
Q_0 value : Strictly returns Q_0/F(r) from the above equation
"""
# Match the LOFAR catalogue to the optical/NIR catalogue
ind_1l, ind_2o, d2d, d3d = search_around_sky(lofar_coords, opt_coords,
radius * u.arcsec)
# Number of LOFAR sources without a optical/NIR counterpart within radius
nl_no_match = len(lofar_coords) - len(np.unique(ind_1l))
# Match this random catalogue to theoptical/NIR catalogue
ind_r, ind_2o, d2d, d3d = search_around_sky(random_coords, opt_coords,
radius * u.arcsec)
# Number of random sources without an optical/NIR counterpart within radius
nr_no_match = len(random_coords) - len(np.unique(ind_r))
q0_value = 1 - (float(nl_no_match) / nr_no_match *
(float(len(random_coords)) / len(lofar_coords)))
return q0_value, nl_no_match, nr_no_match
def test_regression_4082():
"""
Issue: https://github.com/astropy/astropy/issues/4082
"""
from astropy.coordinates import search_around_sky, search_around_3d
cat = SkyCoord([10.076, 10.00455], [18.54746, 18.54896], unit='deg')
search_around_sky(cat[0:1], cat, seplimit=u.arcsec * 60, storekdtree=False)
# in the issue, this raises a TypeError
# also check 3d for good measure, although it's not really affected by this bug directly
cat3d = SkyCoord([10.076, 10.00455]*u.deg, [18.54746, 18.54896]*u.deg, distance=[0.1, 1.5]*u.kpc)
search_around_3d(cat3d[0:1], cat3d, 1*u.kpc, storekdtree=False)
def test_regression_4082():
"""
Issue: https://github.com/astropy/astropy/issues/4082
"""
from astropy.coordinates import search_around_sky, search_around_3d
cat = SkyCoord([10.076, 10.00455], [18.54746, 18.54896], unit='deg')
search_around_sky(cat[0:1], cat, seplimit=u.arcsec * 60, storekdtree=False)
# in the issue, this raises a TypeError
# also check 3d for good measure, although it's not really affected by this bug directly
cat3d = SkyCoord([10.076, 10.00455]*u.deg, [18.54746, 18.54896]*u.deg, distance=[0.1, 1.5]*u.kpc)
search_around_3d(cat3d[0:1], cat3d, 1*u.kpc, storekdtree=False)
def gc_serial(A_coord, B_coord, radius):
'''
'''
from astropy.coordinates import search_around_sky
def assert_input(A_coord, B_coord, radius):
from astropy.coordinates import SkyCoord
from astropy.units import Quantity
assert isinstance(A_coord,SkyCoord), "Was expecting an ~astropy.coordinates.SkyCoord instance for 'A_coord'."
assert isinstance(B_coord,SkyCoord), "Was expecting an ~astropy.coordinates.SkyCoord instance for 'B_coord'."
assert isinstance(radius,Quantity), "Was expecting an ~astropy.units.Quantity instance for 'radius'"
assert_input(A_coord, B_coord, radius)
logging.info("Searching B_coord {1} objects, {0} neighbors.".format(len(B_coord),len(A_coord)))
_prau = A_coord.ra.unit
_pdecu = A_coord.dec.unit
_nrau = B_coord.ra.unit
_ndecu = B_coord.dec.unit
logging.debug("Unit of coordinates being matched: ({0},{1}) and ({2},{3})".format(_prau,_pdecu,_nrau,_ndecu))
match_A_gc_idx, match_B_gc_idx, match_gc_sep, _d3d = search_around_sky(A_coord, B_coord, radius)
from booq.utils import stats
_sts = stats.basic(match_gc_sep.value)
logging.info("Basic stats of distances between matchings: {}".format(_sts))
assert len(match_A_gc_idx) == len(match_B_gc_idx)
return (match_A_gc_idx, match_B_gc_idx, match_gc_sep)
def fill_duplicates(cat, cols):
"""Since duplicates are excluded in the catalogue (to save time),
they must be filled in to create a complete catalogue.
Due to nuances in the flagging routine, this must be done iteratively.
"""
# Fill in `cols` for duplicates by searching for matches in the rest
# of the duplicate components.
# Need to apply this multiple times because of the duplicate flagging algorithm.
missing_comps = cat[(cat.Duplicate_flag >= 1) & np.isnan(cat[cols[0]])]
not_missing_comps = cat[(cat.Duplicate_flag >= 1)
& ~np.isnan(cat[cols[0]])]
missing_coords = SkyCoord(missing_comps["RA"].values,
missing_comps["DEC"].values,
unit=u.deg)
not_missing_coords = SkyCoord(not_missing_comps["RA"].values,
not_missing_comps["DEC"].values,
unit=u.deg)
idx1, idx2, sep, dist = search_around_sky(missing_coords,
not_missing_coords,
seplimit=2 * u.arcsec)
# When multiple matches are found, choose the one with the highest SNR
idx1u, idx1c = np.unique(idx1, return_counts=True)
idx2u = [
idx2[idx1 == i1][0] if i1c == 1 else
idx2[idx1 == i1][final_cat.iloc[idx2[idx1 == i1]]["SNR"].argmax()]
for i1, i1c in zip(idx1u, idx1c)
]
for col in cols:
cat.loc[missing_comps.iloc[idx1].index,
col] = (not_missing_comps[col].iloc[idx2].values)
def estimate_q_m(magnitude, bin_list, n_m, coords_small, coords_big, radius=5):
"""Compute q(m)
Estimation of the distribution of real matched sources with respect
to a magnitude (normalized to 1). As explained in Fleuren et al.
"""
assert len(magnitude) == len(coords_big)
# Cross match
idx_small, idx_big, d2d, d3d = search_around_sky(
coords_small, coords_big, radius*u.arcsec)
n_xm_small = len((idx_small))
idx = np.unique(idx_big)
# Get the distribution of matched sources
n_hist_total, _ = np.histogram(magnitude[idx], bin_list)
n_hist_total = n_hist_total.astype('float')
# Correct probability if there are no sources
if len(magnitude[idx]) == 0:
n_hist_total = np.ones_like(n_hist_total)*0.5
for n in range(len(n_hist_total)):
test = n_hist_total[n]
if (test == 0):
n_hist_total[n] = 0.5
# Estimate real(m)
n_m_nocumsum = np.ediff1d(n_m, to_begin=n_m[0])
real_m = n_hist_total - n_xm_small*n_m_nocumsum*np.pi*(radius)**2
# Remove small negative numbers
real_m[real_m <= 0.] = 0.
real_m_cumsum = np.cumsum(real_m)
return real_m_cumsum/real_m_cumsum[-1]
def estimate_q_m_kde(magnitude, bin_centre, n_m, coords_small, coords_big, radius=5, bandwidth=0.2):
"""Compute q(m)
Estimation of the distribution of real matched sources with respect
to a magnitude (normalized to 1). As explained in Fleuren et al. in a
non-cumulative fashion using a KDE.
For this function we need the centre of the bins instead
of the edges.
"""
assert len(magnitude) == len(coords_big)
# Cross match
idx_small, idx_big, d2d, d3d = search_around_sky(
coords_small, coords_big, radius*u.arcsec)
n_small = len(idx_small)
idx = np.unique(idx_big)
# Get the distribution of matched sources
kde_skl_q_m = KernelDensity(bandwidth=bandwidth)
kde_skl_q_m.fit(magnitude[idx][:, np.newaxis])
pdf_q_m = np.exp(kde_skl_q_m.score_samples(bin_centre[:, np.newaxis]))
n_hist_total = pdf_q_m*len(magnitude[idx])/np.sum(pdf_q_m)
# Correct probability if there are no sources ## CHECK
if len(magnitude[idx]) == 0:
n_hist_total = np.ones_like(n_hist_total)*0.5
# Estimate real(m)
real_m = n_hist_total - n_small*n_m*np.pi*radius**2
# Remove small negative numbers ## CHECK
real_m[real_m <= 0.] = 0.
return real_m/np.sum(real_m)
def plot_one_night(exps, tiledata, night, startdate, center_l=180):
import ephem
from astropy import units as u
from astropy.coordinates import SkyCoord, search_around_sky
from matplotlib import pyplot as p
startmjd = int(
desisurvey.utils.local_noon_on_date(
desisurvey.utils.get_date(startdate)).mjd)
nightnum = night - startmjd
mstarted = (tiledata['PLANNED'] <= nightnum) & (tiledata['PLANNED'] >= 0)
tiles = desisurvey.tiles.get_tiles()
p.clf()
p.subplots_adjust(hspace=0)
p.subplots_adjust(left=0.1, right=0.9)
programs = ['DARK', 'BRIGHT']
expindex = tiles.index(exps['TILEID'])
expnight = exps['MJD'].astype('i4')
m = expnight == night
medianmjd = np.median(exps['MJD'][m])
mayall = ephem.Observer()
config = desisurvey.config.Configuration()
coord = SkyCoord(ra=tiles.tileRA * u.deg, dec=tiles.tileDEC * u.deg)
mayall.lon = config.location.longitude().to(u.radian).value
mayall.lat = config.location.latitude().to(u.radian).value
mayall.date = medianmjd + (2400000.5 - 2415020)
moon = ephem.Moon()
moon.compute(mayall)
tile_diameter = config.tile_radius() * 2
for i, prog in enumerate(programs):
mprog = prog == tiles.tileprogram
mprogstarted = mstarted & mprog
p.subplot(len(programs), 1, i + 1)
ra = ((tiles.tileRA - (center_l - 180)) % 360) + (center_l - 180)
p.plot(ra[mprog],
tiles.tileDEC[mprog],
'.',
color='gray',
markersize=1)
p.plot(ra[mprogstarted],
tiles.tileDEC[mprogstarted],
'.',
color='green',
markersize=5)
m = (expnight == night) & (tiles.tileprogram[expindex] == prog)
p.plot(ra[expindex[m]], tiles.tileDEC[expindex[m]], 'r-+')
idx1, idx2, sep2d, dist3d = search_around_sky(coord[expindex[m]],
coord[expindex[m]],
tile_diameter * 10)
mdiff = expindex[m][idx1] != expindex[m][idx2]
if np.sum(mdiff) > 0:
print(f'min separation {prog}: {np.min(sep2d[mdiff])}')
p.gca().set_aspect('equal')
p.plot(((np.degrees(moon.ra) - (center_l - 180)) % 360) +
(center_l - 180),
np.degrees(moon.dec),
'o',
color='yellow',
markersize=10,
markeredgecolor='black')
def __call__(self, radius=None):
"""Compute the Q_0 for a given radius (in arcsecs)"""
if radius is None:
radius = self.radius
# Generate random catalogue with n sources as the small one
random_small = self.field.random_catalogue(self.n_small)
idx_random_small, idx_big, d2d, d3d = search_around_sky(
random_small, self.coords_big, radius*u.arcsec)
nomatch_random = self.n_small - len(np.unique(idx_random_small))
print('number of random sources with no matches: {:4d}'.format(nomatch_random))
# Compute match in radius
idx_small, idx_big, d2d, d3d = search_around_sky(
self.coords_small, self.coords_big, radius*u.arcsec)
nomatch_small = self.n_small - len(np.unique(idx_small))
print('number of real sources with no matches: {:4d}'.format(nomatch_small))
return (1. - float(nomatch_small)/float(nomatch_random))
def join_table_by_coordinates(table,
table_to_join,
columns_to_join=None,
columns_to_rename=None,
max_distance=1.0 / 3600.0,
missing_value=np.nan,
table_ra_name='RA',
table_dec_name='DEC',
table_to_join_ra_name='RA',
table_to_join_dec_name='DEC',
unit='deg'):
"""
join two table by matching the sky coordinates
Examples
--------
wise_cols = ('W1_MAG', 'W1_MAG_ERR', 'W2_MAG', 'W2_MAG_ERR')
cols_rename = {'W1_MAG':'W1', 'W1_MAG_ERR':'W1ERR', 'W2_MAG':'W2', 'W2_MAG_ERR':'W2ERR'}
join_table_by_coordinates(base, wise, wise_cols, cols_rename)
"""
t1 = table
t2 = table_to_join
ra1 = table_ra_name
dec1 = table_dec_name
ra2 = table_to_join_ra_name
dec2 = table_to_join_dec_name
idx1, idx2 = search_around_sky(SkyCoord(t1[ra1], t1[dec1], unit=unit),
SkyCoord(t1[ra1], t1[dec1], unit=unit),
Quantity(max_distance, unit=unit))[:2]
n_matched = len(idx1)
if n_matched:
if columns_to_join is None:
columns_to_join = t2.colnames
if columns_to_rename is None:
columns_to_rename = dict()
if isinstance(missing_value, dict):
missing_value_dict = missing_value
missing_value = np.nan
else:
missing_value_dict = dict()
for c2 in columns_to_join:
c1 = columns_to_rename.get(c2, c2)
if c1 not in t1:
t1[c1] = missing_value_dict.get(c1, missing_value)
t1[c1][idx1] = t2[c2][idx2]
return n_matched
def main(
sample,
outfile,
img_size,
img_path,
threads=None,
parallel=True,
all_comps=None,
seplimit=90,
**kwargs,
):
kwargs.update(
dict(
ir=True,
img_size=img_size,
radio_path=img_path,
ir_path=img_path,
all_comps=all_comps,
))
if all_comps is not None:
seplimit = u.Quantity(seplimit, u.arcsec)
coords = SkyCoord(np.array(sample["RA"]),
np.array(sample["DEC"]),
unit=u.deg)
all_coords = SkyCoord(np.array(all_comps["RA"]),
np.array(all_comps["DEC"]),
unit=u.deg)
idx1, idx2, sep, dist = search_around_sky(coords,
all_coords,
seplimit=seplimit)
kwargs.update(comp_idx_map=(idx1, idx2))
with pu.ImageWriter(outfile, 0, img_size, clobber=True) as pk_img:
if not parallel:
for idx in tqdm(sample.index):
out = vlass_preprocessing(idx, sample, **kwargs)
if out is not None:
pk_img.add(out[1], attributes=out[0])
else:
if threads is None:
threads = cpu_count()
pool = Pool(processes=threads)
results = [
pool.apply_async(vlass_preprocessing,
args=(idx, sample),
kwds=kwargs) for idx in sample.index
]
for res in tqdm(results):
out = res.get()
if out is not None:
pk_img.add(out[1], attributes=out[0])
def getPlates(plateFile,cat):
plChunk=np.loadtxt(plateFile,skiprows=41,usecols=(3,4))
catPlates=coord.SkyCoord(plChunk[:,0]*u.degree,plChunk[:,1]*u.degree)
catObj=coord.SkyCoord(cat[:,0]*u.degree,cat[:,1]*u.degree)
objIndex,plateIndex,sep,dist=coord.search_around_sky(catObj,catPlates,1.49*u.degree)
# print('objIndex:',objIndex,'plateIndex:',plateIndex)
# print('lencat',len(catObj),'objIndex',len(objIndex),'platejIndex',len(plateIndex))
objId,counts=np.unique(objIndex,return_counts=True)
# print(len(catObj),len(catPlates))
# print(len(objId),objId,counts)
val,countsC=np.unique(counts,return_counts=True)
print("objects belonging to",val,"plates, fibers:",countsC)
return counts
def overlay_points(neuron: tuple, ed: pu.heatmap, trans: pu.transform,
srcs: Table, cata: Table, *args, **kwargs):
"""Produce a list of (x,y) of offsets sources that are near
objects for each source matching up to a neuron.
Arguments:
neuron {tuple} -- BMU position matching to
ed {pu.heatmap} -- Mapped similarity
trans {pu.transform} -- Transform matrix for the similarity
srcs {Table} -- Catalogue of sources matching those in ed/trans
cata {Table} -- Catalogue of sources to overlay
"""
head = som.file_head
target_idx = neuron[1] * head[1] + neuron[0]
print("\tSearching for matches")
pos_min = np.argmin(ed.data.reshape(ed.data.shape[0], -1), axis=1)
mask = np.argwhere(pos_min == target_idx)
pix_pos = []
print(f"\tFound {mask.shape}")
print("\tIterating and looking for object")
for i in mask:
row = srcs[i]
trans_info = trans.get_neuron_transform(index=i, pos=neuron)
# Added to test the individual flipping cases to see what a
# problem was. Still not covinced that it is completely correct.
# Got to do with a transpose from splitting/concating PINK binaries.
# if trans_info[0] == 0:
spos = SkyCoord(ra=row['RA'] * u.deg, dec=row['DEC'] * u.deg)
res = search_around_sky(spos, cata, seplimit=5 * u.arcmin)
for idx in res[1]:
offsets = pu.great_circle_offsets(spos,
cata[idx],
pix_scale=1.5 * u.arcsec)
offsets = apply_transform(trans_info, (offsets, ))
pix_pos.append(offsets)
pix_pos = np.array(pix_pos).reshape(
-1, 2) # No idea why the reshape is needed.
return pix_pos, mask
def refmatch(self, sourceframe):
from astropy.coordinates import SkyCoord, search_around_sky # @UnresolvedImport
from astropy import units as u
ra1 = sourceframe.stars['alphawin_j2000']
dec1 = sourceframe.stars['deltawin_j2000']
coords = SkyCoord(ra=ra1 * u.degree, dec=dec1 * u.degree)
fluxes = sourceframe.stars['flux_auto'] / sourceframe.expt
bidx, refidx, _, _ = search_around_sky(coords, self.refcoord,
0.6 * u.arcsec)
# create an empty arry and set with nans
compfluxes = np.ones(self.nstars) * np.nan
compfluxes[refidx] = fluxes[bidx]
return compfluxes
def __init__(
self,
base_catalogue: SkyCoord,
match_catalogues: Tuple[SkyCoord, ...],
annotation: Annotator,
som_set: SOMSet,
cpu_cores: int = None,
seplimit: Angle = 1 * u.arcminute,
progress: bool = False,
**ct_kwargs,
):
"""Create a set of Filters that describe the projection of sources onto their neurons. Other keyword-arguments are passed to `CoordinateTransformer`
Arguments:
base_catalogue {SkyCoord} -- The central positions of each image that corresponds to the Mapping and Transform of the currect `som_set`
match_catalogues {Tuple[SkyCoord,...]} -- Catalogues of positions to project the neurons through. Each catalogue is assumed to correspond to a single channel
annotation {Annotator} -- Previously defined annotated SOM
som_set {pu.SOMSet} -- Reference to the SOM, Mapping and Transform binary files for the projection
Keyword Arguments:
cpu_cores {int} -- The number of CPU cores to use while projecting the filters. The default is to use one (and avoid the ProcessPoolExecutor) (default: {None})
seplimit {Angle} -- Matching area for the `search_around_sky` matcher (defaul: {1*astropy.units.arcminute})
progress {bool} -- Enable the `tqdm` progress bar updates (default: {False})
"""
self.base_catalogue = base_catalogue
self.annotation = annotation
self.som_set = som_set
self.cpu_cores = cpu_cores
self.seplimit = seplimit
self.progress = progress
self.ct_kwargs = ct_kwargs
assert isinstance(
match_catalogues, tuple
), f"Expect tuple of SkyCoord catalogues, even if only of length 1. Received object of type {type(match_catalogues)}"
self.match_catalogues = match_catalogues
self.sky_matches = [
search_around_sky(self.base_catalogue, mc, self.seplimit)
for mc in self.match_catalogues
]
logger.debug(
f"`search_around_sky` completed, number of matches {[len(sk[0]) for sk in self.sky_matches]}"
)
self.filters = self.project()
def get_q_m(lofar_coords, opt_coords, rmax, magnitude, bin_list, area, n_m):
"""
Get q(m)/Q_0 - computes the real(m)/np.cumsum(real(m))
Parameters:
-----------
lofar_coords : LOFAR coordinates (SkyCoord object)
opt_coords : Optical/NIR coordinates (SkyCoord object)
rmax : rmax within which to search for optical sources of a LOFAR source
magnitude : magnitude distribution
bin_list : List of bin edges - THE SAME BINS USED TO COMPUTE n(m)
area : Survey area of the optical/NIR band
n_m : output of get_n_m --- n(m) distribution
Returns:
--------
q(m): Strictly, it returns q(m)/Q_0
"""
# Search for all opt/NIR sources within rmax of LOFAR sources
ind_l, ind_o, s2d, d3d = search_around_sky(lofar_coords, opt_coords,
rmax * u.arcsec)
# Get the unique indices into opt_coords that match
ind_o_un = np.unique(ind_o)
ind_l_un = np.unique(ind_l)
print("Unique opt_ind: ", len(ind_o_un))
print("Unique LOFAR ind: ", len(ind_l_un))
# Get the magnitude of the optical/NIR sources that have a match - defined as total(m)
total_m, _ = np.histogram(magnitude[ind_o_un], bin_list)
# Compute real(m)
real_m = np.cumsum(total_m) - len(lofar_coords) * n_m * np.pi * rmax**2
# Remove any possible small negative values?
real_m[real_m <= 0.] = 0.
# Cumulative sum of the real_m
# real_m_cumsum = np.cumsum(real_m)
# Return the q(m)/Q_0 value - where Q_0 has yet to be determined (using an iterative approach)
return real_m / real_m[-1], np.cumsum(total_m), real_m
def apply_circles(self, lon=None, lat=None):
# read the list of circles from the file
try:
hdul = fits.open(self.filename)
except:
raise Exception(f"Failed to find/open circles file: {self.filename}")
data = hdul[1].data
try:
circle_lon = data[self.col_lon]
circle_lat = data[self.col_lat]
except:
raise Exception(f"Failed to find circle columns: {self.filename}[1][{self.col_lon},{self.col_lat}]")
if isinstance(self.radius, float):
circle_radius = np.full(len(data), self.radius)
elif isinstance(self.radius, str):
try:
circle_radius = data[self.radius]
except:
raise Exception(f"Failed to find circle column: {self.filename}[1][{self.radius}]")
else:
raise Exception(f"Cannot interpret radius: {self.radius}")
hdul.close()
### need to use a single radius for all objects
seplimit = u.Quantity(circle_radius.max(), unit='deg')
coords_c = SkyCoord(circle_lon, circle_lat, frame="icrs", unit="deg")
coords_t = SkyCoord(lon, lat, frame="icrs", unit="deg")
idx_t, idx_c, d2d, d3d = search_around_sky(coords_t, coords_c, seplimit)
## filter the list of matches to use the per-circle radius
i_t_ok = [ i_t for i_t,i_c,d in zip(idx_t, idx_c, d2d) if d.deg < circle_radius[i_c]]
m_mask = np.zeros(len(lon), np.bool)
m_mask[i_t_ok] = True
return m_mask
def indices_xfind_coords(
coords1,
coords2,
maxdist=1 * u.arcsec,
obstime=None,
):
"""Indices of X-Find coords.
Returns
-------
idx1 : integer array
idx2 : integer array
indices into `other` for all coordinates which have a "match"
in `catalog`.
info : dict
Useful information # TODO
See Also
--------
:func:`~astropy.coordinates.search_around_sky`
:func:`~astropy.coordinates.search_around_3d`
"""
if u.get_physical_type(maxdist.unit) == "angle":
idx1, idx2, sep2d, dist3d = coord.search_around_sky(
coords1,
coords2,
seplimit=maxdist,
)
elif u.get_physical_type(maxdist.unit) == "length":
idx1, idx2, sep2d, dist3d = coord.search_around_3d(
coords1,
coords2,
distlimit=maxdist,
)
info = {"sep2d": sep2d, "dist3d": dist3d}
return idx1, idx2, info
def coord_matching(first_coords, second_coords, match_sep):
"""
Performs a coordinates cross-match of the positions between two SkyCoord objects
coord_matching(tomatch_coords, catalog_coords, match_sep)
Input:
first_coords: First set of coordinates - a SkyCoord object
second_coords: Second set of coordinates - a SkyCoord object
match_sep: Separation to search within - in arcseconds
Output:
indx1: Indices into first_coords that have matched with coords2 - Array
indx2: Indices into second_coords that have matched with coords1 - Array
sep2d: On-sky separation between the coordinates - (Angle object)
"""
# Perform the catalog match
indx1, indx2, sep2d, dist3d = search_around_sky(first_coords,
second_coords,
seplimit=match_sep *
u.arcsec)
return indx1, indx2, sep2d
def best_host(components, ir_srcs, sep_limit):
# A host is most likely when it aligns with a radio components
likely_host = False
sep_limit = u.Quantity(sep_limit, u.arcsec)
sky_matches = search_around_sky(components,
ir_srcs,
seplimit=15 * u.arcsecond)
if len(sky_matches[0]) > 0:
best_match = np.argmin(sky_matches[2])
best_ir_match = ir_srcs[sky_matches[1][best_match]]
# Obtained from cross-matching a shifted catalogue to WISE
if sky_matches[2][best_match] < 3.4 * u.arcsecond:
# ax.scatter(
# best_ir_match.ra,
# best_ir_match.dec,
# transform=ax.get_transform("world"),
# color="white",
# s=22,
# )
likely_host = True
return likely_host
# Reading SDSS Stripe 82 clean data
S82 = pd.read_csv('../../data/clean_stripe82.csv')
# Reading SDSS DR7 clean data
DR7 = pd.read_csv('../../data/clean_dr7.csv')
# Rename the ra, dec coordinates from the 2 different data sets
DR7 = DR7.rename(index=str, columns={'ra': 'ra_DR7', 'dec': 'dec_DR7'})
S82 = S82.rename(index=str, columns={'ra': 'ra_S82', 'dec': 'dec_S82'})
# Match data attributes in the 2 data sets using astropy's SkyCoord
COORD1 = SkyCoord(DR7['ra_DR7'], DR7['dec_DR7'], frame='icrs', unit='deg')
COORD2 = SkyCoord(S82['ra_S82'], S82['dec_S82'], frame='icrs', unit='deg')
IDX1, IDX2, OTHER1, OTHER2 = search_around_sky(COORD1,
COORD2,
seplimit=0.5 * u.arcsec)
# Generating columns for the matched
X_TRAIN = []
for i in range(len(IDX1)):
result = DR7.iloc[IDX1[i]].append(S82.iloc[IDX2[i]])
X_TRAIN.append(result)
X_TRAIN = pd.concat(X_TRAIN, axis=1)
X_TRAIN = X_TRAIN.T
# Remove the unnecessary part
X_TRAIN = X_TRAIN.loc[:, ~X_TRAIN.columns.str.contains('^Unnamed')]
X_TRAIN = X_TRAIN.drop(['BH_mass'], axis=1)
X_TRAIN = X_TRAIN.astype({'ID': int})
def xmatch_cat(table1=None, table2=None,
radec1=None, radec2=None,
nthneighbor=None,
multimatch=False,
seplimit=10.0,
selfmatch=False,
colnames_radec1=['ra', 'dec'],
colnames_radec2=['ra', 'dec'],
units_radec1=['degree', 'degree'],
units_radec2=['degree', 'degree'],
stats=False,
debug=False,
verbose=False,
method=False):
"""RA, Dec nearest xmatch for two lists; returns pointers
nearest match
input can be an astropy table or zipped radec as a list
e.g.
c = zip([1],[1])
radec1 = zip(ra1 , dec1)
radec1 = np.column_stack(ra1, dec1))
Self match notes:
"""
import numpy as np
import matplotlib.pyplot as plt
from astropy.table import Table, hstack
from astropy.coordinates import SkyCoord
from astropy.coordinates import search_around_sky, match_coordinates_sky
from astropy import units as u
from astropy.stats import mad_std, median_absolute_deviation
if verbose or debug:
print('__file__:', __file__)
print('__name__:', __name__)
try:
if 'filename' in table1.meta:
print('table1.filename:', table1.meta['filename'])
except:
print("table1 has no metadata or table1.meta['filename']")
if verbose or debug:
print('colnames_radec1:', colnames_radec1)
table1.info()
# selfmatch does not need a 2nd table
if not selfmatch:
try:
if 'filename' in table2.meta:
print('table2.filename:', table2.meta['filename'])
except:
print("table2 has no metadata or table2.meta['filename']")
if verbose or debug:
print('colnames_radec2:', colnames_radec2)
table2.info()
if selfmatch:
table2 = table1
colnames_radec2 = colnames_radec1
if nthneighbor is None:
nthneighbor = 2
if nthneighbor is None:
nthneighbor = 1
ra1 = table1[colnames_radec1[0]]
dec1 = table1[colnames_radec1[1]]
if verbose or debug:
print('table1: ', colnames_radec1[0], table1[colnames_radec1[0]].unit)
print('table1: ', colnames_radec1[1], table1[colnames_radec1[1]].unit)
ra2 = table2[colnames_radec2[0]]
dec2 = table2[colnames_radec2[1]]
if verbose or debug:
print('table2: ', colnames_radec2[0], table2[colnames_radec2[0]].unit)
print('table2: ', colnames_radec2[1], table2[colnames_radec2[1]].unit)
if stats or verbose or debug:
print('RA1 range:', np.min(ra1), np.max(ra1))
print('Dec1 range:', np.min(dec1), np.max(dec1))
print('RA1 range:', np.min(ra2), np.max(ra2))
print('Dec1 range:', np.min(dec2), np.max(dec2))
skycoord1 = SkyCoord(ra1, dec1, unit=units_radec1, frame='icrs')
skycoord2 = SkyCoord(ra2, dec2, unit=units_radec2, frame='icrs')
# idx is an integer array into the second cordinate array to get the
# matched points for the second coordindate array.
# Shape of idx matches the first coordinate array
idx1 = []
idx2 = []
if not method:
if not multimatch:
idx2, d2d, d3d = \
match_coordinates_sky(skycoord1,
skycoord2,
nthneighbor=nthneighbor)
if multimatch:
idx1, idx2, d2d, d3d = \
search_around_sky(skycoord1,
skycoord2,
seplimit * u.arcsec)
# alternative 'method' form
if method:
if not multimatch:
idx2, d2d, d3d = \
skycoord1.match_to_catalog_sky(skycoord2,
nthneighbor=nthneighbor)
if multimatch:
idx1, idx2, d2d, d3d = \
skycoord1.search_around_sky(skycoord2,
seplimit * u.arcsec)
# compute the separations and
if not multimatch:
separation = skycoord1.separation(skycoord2[idx2])
dra, ddec = \
skycoord1.spherical_offsets_to(skycoord2[idx2])
if multimatch:
separation = skycoord1[idx1].separation(skycoord2[idx2])
dra, ddec = \
skycoord1[idx1].spherical_offsets_to(skycoord2[idx2])
if stats or verbose or debug:
print('multimatch:', multimatch)
print('seplimit:', seplimit)
print('len(table1):', len(table1))
print('len(table2):', len(table2))
print('len(idx1):', len(idx1))
print('len(idx2):', len(idx2))
print('idxmatch range:', np.min(idx2), np.max(idx2))
print('d2d range:', np.min(d2d), np.max(d2d))
print('d2d range:', np.min(d2d).arcsec, np.max(d2d).arcsec)
print('d2d median:', np.median(d2d).arcsec)
median_separation = np.median(separation).arcsec
mad_std_separation = mad_std(separation.arcsec)
print('dR range (arcsec):',
np.min(separation.arcsec), np.max(separation.arcsec))
print('dR mean, std (arcsec):',
np.mean(separation).arcsec, np.std(separation).arcsec)
print('dR median, mad_std (arcsec):',
median_separation, mad_std_separation)
print()
median_dra = np.median(dra).arcsec
mad_std_dra = mad_std(dra.arcsec)
print('dRA min, max:',
np.min(dra).arcsec, np.max(dra).arcsec)
print('dRA mean, std:',
np.mean(dra).arcsec, np.std(dra).arcsec)
print('dRA median, mad_std:',
median_dra, mad_std_dra)
print()
median_ddec = np.median(ddec).arcsec
mad_std_ddec = mad_std(ddec.arcsec)
print('dDec min, max:',
np.min(ddec).arcsec, np.max(ddec).arcsec)
print('dDec mean, std:',
np.mean(ddec).arcsec, np.std(ddec).arcsec)
print('dDec median, mad_std:',
median_ddec, mad_std_ddec)
print()
# convert to arcsec for convenience
separation = separation.arcsec
dr = d2d.arcsec
dra = dra.arcsec
ddec = ddec.arcsec
# return dra, ddec, dr in arcsec
# as a list or could be dict; check if scales from 10^3 -> 10^6 -> 10^9
drplus = [dra, ddec, dr]
if debug or verbose:
print(len(idx2), len(dr))
print(len(drplus), len(drplus[0]), len(drplus[1]), len(drplus[2]))
# could add option to return dr, dra, ddec
if not multimatch:
return idx2, dr, dra, ddec
if multimatch:
return (idx1, idx2), dr, dra, ddec
def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir):
# pylint: disable=no-member
# Try to read cosmology from catalog, otherwise defualts to WMAP7
try:
cosmo = catalog_instance.cosmology
except AttributeError:
cosmo = WMAP7
# Create interpolation tables for efficient computation of sigma crit
z = np.linspace(0, self.zmax, int(self.zmax)*100)
d1 = cosmo.angular_diameter_distance(z) # in Mpc
angular_diameter_distance = interp1d(z, d1, kind='quadratic')
d2 = cosmo.comoving_transverse_distance(z) # in Mpc
comoving_transverse_distance = interp1d(z, d2, kind='quadratic')
# Now figure out the lenses, for the validation data available,
# each have slightly non-trivial cuts, so we do them separately... not totally ideal
if self.data == 'sdss_lowz':
# Singh et al (2015) (http://adsabs.harvard.edu/abs/2015MNRAS.450.2195S) measurements on the SDSS LOWZ sample.
res = catalog_instance.get_quantities(['redshift_true', 'ra', 'dec', 'shear_1', 'shear_2',
'mag_true_i_sdss', 'mag_true_z_sdss','mag_true_g_sdss', 'mag_true_r_sdss'])
# Compute mask for lowz sample
# These cuts are defined in section 3 of https://arxiv.org/pdf/1509.06529.pdf
# and summarised here: http://www.sdss.org/dr14/algorithms/boss_galaxy_ts/#TheBOSSLOWZGalaxySample
# Definition of auxiliary colors:
cperp = (res['mag_true_r_sdss'] - res['mag_true_i_sdss']) - (res['mag_true_g_sdss'] - res['mag_true_r_sdss'])/4.0 - 0.18
cpar = 0.7*(res['mag_true_g_sdss'] - res['mag_true_r_sdss']) + 1.2*((res['mag_true_r_sdss'] - res['mag_true_i_sdss'])-0.18)
# LOWZ selection cuts:
mask_lens = np.abs(cperp) < 0.2 # color boundaries
mask_lens &= res['mag_true_r_sdss'] < (13.5 + cpar/0.3) # sliding magnitude cut
mask_lens &= (res['mag_true_r_sdss'] > 16) &(res['mag_true_r_sdss'] < 19.6)
# Additional redshift cuts used in Singh et al. (2015)
mask_lens &= (res['redshift_true'] > self.zmin_l) & (res['redshift_true'] < self.zmax_l)
Mask_lens = [mask_lens]
fig = plt.figure()
if self.data == 'cfhtlens':
res = catalog_instance.get_quantities(['redshift_true', 'ra', 'dec', 'shear_1', 'shear_2',
'Mag_true_g_lsst_z0', 'Mag_true_r_lsst_z0'])
Mr_min = np.array([-21.0,-22.0,-23.0,-24.0])
Mr_max = np.array([-20.0,-21.5,-22.5,-23.5])
blue_frac = np.array([0.7,0.32,0.11,0.03])*100
gr = res['Mag_true_g_lsst_z0'] - res['Mag_true_r_lsst_z0'] # larger number means redder
Mask_lens = []
for i in range(4):
mask_lens = (res['redshift_true']>self.zmin_l) & (res['redshift_true']<self.zmax_l) & (res['Mag_true_r_lsst_z0']>Mr_min[i]) & (res['Mag_true_r_lsst_z0']<Mr_max[i])
gr_threshold = np.percentile(gr[mask_lens], blue_frac[i])
Mask_lens.append(mask_lens & (gr>gr_threshold))
Mask_lens.append(mask_lens & (gr<gr_threshold))
fig1 = plt.figure(1, figsize=(12,9))
fig2 = plt.figure(2, figsize=(12,5))
if self.data == 'sdss_main':
res = catalog_instance.get_quantities(['redshift_true', 'ra', 'dec', 'shear_1', 'shear_2',
'mag_true_i_sdss', 'mag_true_z_sdss','mag_true_g_sdss', 'mag_true_r_sdss', 'stellar_mass_bulge', 'stellar_mass_disk','Mag_true_g_sdss_z0','Mag_true_r_sdss_z0'])
gr = res['Mag_true_g_sdss_z0'] - res['Mag_true_r_sdss_z0'] # larger number means redder
sm = res['stellar_mass_bulge'] + res['stellar_mass_disk']
SM_min = np.array([10,10.7,11.2,11.6])
SM_max = np.array([10.4,11.0,11.4,15.0])
Mask_lens = []
for i in range(4):
mask_lens = (res['redshift_true']>self.zmin_l) & (res['redshift_true']<self.zmax_l) & (res['mag_true_r_sdss']< 17.7) & (np.log10(sm)>SM_min[i]) & (np.log10(sm)<SM_max[i])
Mask_lens.append(mask_lens & (gr>0.7)) # for the data, 0.7 is used for k-correct colors at z=0.1
Mask_lens.append(mask_lens & (gr<0.7))
fig1 = plt.figure(1, figsize=(12,9))
fig2 = plt.figure(2, figsize=(12,5))
# Computing mask for source sample, this only serves to keep the number of galaxies managable
mask_source = (res['redshift_true'] > self.zmin_s) & (res['redshift_true'] < self.zmax_s)
inds = np.where(mask_source)[0]
if len(inds) > int(self.max_background_galaxies):
mask_source[inds[np.random.choice(len(inds),
size=len(inds) - int(self.max_background_galaxies),
replace=False)]] = False
coords = SkyCoord(ra=res['ra']*u.degree, dec=res['dec']*u.degree)
coords_s = coords[mask_source]
# run gammat in thin redshift bins, loop over lens bins of different stellar mass and colors
for i in range(len(Mask_lens)):
nlens = len(np.where(Mask_lens[i])[0]) / catalog_instance.sky_area
with open(os.path.join(output_dir, 'galaxy_density_'+str(self.data)+'.dat'), 'a') as f:
f.write('{} \n'.format(nlens))
# Create astropy coordinate objects
coords_l = coords[Mask_lens[i]]
# Search for neighbours
idx1, idx2, sep2d, _ = search_around_sky(coords_l, coords_s, 3.*u.deg)
# Computing sigma crit for each pair
zl = res['redshift_true'][Mask_lens[i]][idx1]
zs = res['redshift_true'][mask_source][idx2]
# Warning: this assumes a flat universe
# See http://docs.astropy.org/en/v0.3/_modules/astropy/cosmology/core.html#FLRW.angular_diameter_distance_z1z2
dm1 = comoving_transverse_distance(zl)
dm2 = comoving_transverse_distance(zs)
angular_diameter_distance_z1z2 = u.Quantity((dm2 - dm1)/(1. + zs), u.Mpc)
sigcrit = cst.c**2 / (4.*np.pi*cst.G) * angular_diameter_distance(zs) / \
((1. + zl)**2. * angular_diameter_distance_z1z2 * angular_diameter_distance(zl))
# NOTE: the validation data is in comoving coordinates, the next few
# lines take care of proper unit conversions
# Apply unit conversion to obtain sigma crit in h Msol /pc^2 (comoving)
cms = u.Msun / u.pc**2
sigcrit = sigcrit*(u.kg/(u.Mpc* u.m)).to(cms) / cosmo.h
# Computing the projected separation for each pairs, in Mpc/h (comoving)
r = sep2d.rad*angular_diameter_distance(zl)*(1. + zl) * cosmo.h
# Computing the tangential shear
thetac = np.arctan2((coords_s[idx2].dec.rad - coords_l[idx1].dec.rad) / np.cos((coords_s[idx2].dec.rad + coords_l[idx1].dec.rad) / 2.0),coords_s[idx2].ra.rad - coords_l[idx1].ra.rad)
gammat = -(res['shear_1'][mask_source][idx2] * np.cos(2*thetac) - res['shear_2'][mask_source][idx2] * np.sin(2*thetac))
# Binning the tangential shear
bins = np.logspace(np.log10(self.Rmin), np.log10(self.Rmax), self.nR, endpoint=True)
counts = np.histogram(r, bins=bins)[0]
gt, b = np.histogram(r, bins=bins, weights=gammat*sigcrit)
rp = 0.5*(b[1:]+b[:-1])
gt = gt/counts
outfile = os.path.join(output_dir, 'DS_'+str(self.data)+'_'+str(i)+'.dat')
np.savetxt(outfile, np.vstack((rp, gt)).T)
if self.data == 'sdss_lowz':
ax = plt.subplot(111)
plt.errorbar(self.validation_data[:,0], self.validation_data[:,1], yerr=self.validation_data[:,2], label='SDSS LOWZ from Singh et al. (2015)',c='k', lw=1, marker='.', fmt='.', capthick=0.8, capsize=2.2)
plt.loglog(rp, gt, label=catalog_name)
plt.title('Lens number density: '+str(nlens)[:4]+' per sq. deg')
ax.set_xlabel('$r_p$ [Mpc/h]')
ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$')
ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3)
ax.set_ylim(0.5, 100)
if self.data == 'cfhtlens':
ii = np.mod(i,2)
iii = int(i/2)
plt.figure(1)
ax = plt.subplot(2,2,iii+1)
if ii==0:
plt.loglog(rp, gt, label=str(Mr_min[int(i/2)])+'< Mr < '+str(Mr_max[int(i/2)])+'; red; '+catalog_name, lw=2, color='r', alpha=0.5)
plt.errorbar(self.validation_data[:,0]/1000*(7./10.), self.validation_data[:,iii*2+1]/(7./10.), color='darkred', lw=2, marker='x', fmt='.', label='Velander et al. (2013)')
plt.text(self.Rmin*0.7*1.5, 1.5,'Red: '+str(nlens)[:4]+' per sq. deg')
else:
plt.loglog(rp, gt, label=str(Mr_min[int(i/2)])+'< Mr < '+str(Mr_max[int(i/2)])+'; blue', lw=2, color='b', alpha=0.5)
plt.errorbar(self.validation_data[:,0]/1000*(7./10.), self.validation_data[:,iii*2+2]/(7./10.), color='darkblue', lw=2, marker='x', fmt='.')
plt.title('Lens number density: '+str(nlens)[:4]+' per sq. deg')
plt.text(self.Rmin*0.7*1.5, 1.0,'Blue: '+str(nlens)[:4]+' per sq. deg')
ax.legend()
ax.set_xlabel('$r_p$ [Mpc/h]')
ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$')
ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3)
ax.set_ylim(0.5, 1000)
plt.tight_layout()
plt.figure(2)
ax = plt.subplot(1,2,ii+1)
plt.loglog(rp, gt, label='['+str(Mr_min[int(i/2)])+', '+str(Mr_max[int(i/2)])+']')
if ii==0:
plt.title('red')
else:
plt.title('blue')
if i==(len(Mask_lens)-1):
plt.legend()
ax.set_xlabel('$r_p$ [Mpc/h]')
ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$')
ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3)
ax.set_ylim(0.5, 500)
if self.data=='sdss_main':
ii = np.mod(i,2)
iii = int(i/2)
plt.figure(1)
ax = plt.subplot(2,2,iii+1)
if ii==0:
plt.loglog(rp, gt, label=str(SM_min[int(i/2)])+'< log10(M*) < '+str(SM_max[int(i/2)])+'; red; '+catalog_name, lw=2, color='r', alpha=0.5)
plt.errorbar(self.validation_data[:15,0], self.validation_data[ii*15:(ii+1)*15,int(i/2)*4+1], yerr=self.validation_data[ii*15:(ii+1)*15,int(i/2)*4+2], color='darkred', lw=2, marker='x', fmt='.', label='Mandelbaum et al. (2016)')
plt.text(self.Rmin*0.7*1.5, 1.5,'Red: '+str(nlens)[:4]+' per sq. deg')
else:
plt.loglog(rp, gt, label=str(SM_min[int(i/2)])+'< log10(M*) < '+str(SM_max[int(i/2)])+'; blue', lw=2, color='b', alpha=0.5)
plt.errorbar(self.validation_data[:15,0], self.validation_data[ii*15:(ii+1)*15,int(i/2)*4+1], yerr=self.validation_data[ii*15:(ii+1)*15,int(i/2)*4+2], color='darkblue', lw=2, marker='x', fmt='.')
plt.text(self.Rmin*0.7*1.5, 1,'Blue: '+str(nlens)[:4]+' per sq. deg')
ax.legend()
ax.set_xlabel('$r_p$ [Mpc/h]')
ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$')
ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3)
ax.set_ylim(0.5, 1000)
plt.tight_layout()
plt.figure(2)
ax = plt.subplot(1,2,ii+1)
plt.loglog(rp, gt, label='['+str(SM_min[int(i/2)])+', '+str(SM_max[int(i/2)])+']')
if ii==0:
plt.title('red')
else:
plt.title('blue')
if i==(len(Mask_lens)-1):
plt.legend()
ax.set_xlabel('$r_p$ [Mpc/h]')
ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$')
ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3)
ax.set_ylim(0.5, 500)
plt.tight_layout()
print(self.data)
if self.data=='cfhtlens' or self.data=='sdss_main':
fig1.savefig(os.path.join(output_dir, 'delta_sigma_'+str(catalog_name)+'1.png'))
plt.close(fig1)
fig2.savefig(os.path.join(output_dir, 'delta_sigma_'+str(catalog_name)+'2.png'))
plt.close(fig2)
else:
fig.savefig(os.path.join(output_dir, 'delta_sigma_'+str(catalog_name)+'.png'))
plt.close(fig)
return TestResult(inspect_only=True)
cat_neighbors = cat_neighbors[cat_neighbors['sSFR_med'] > -11]
elif pair_sfq == 'q-q':
cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sSFR_med'] < -11]
cat_neighbors = cat_neighbors[cat_neighbors['sSFR_med'] < -11]
coord_neighbors = SkyCoord(
np.array(cat_neighbors['ra']) * u.deg,
np.array(cat_neighbors['dec']) * u.deg)
coord_gals_z = SkyCoord(
np.array(cat_neighbors_z['ra']) * u.deg,
np.array(cat_neighbors_z['dec']) * u.deg)
if len(cat_neighbors_z) > 5:
len_cat_nei.append(len(cat_neighbors_z))
arr = search_around_sky(coord_gals_z, coord_neighbors,
10. / 3600 * u.deg)
arr_all = search_around_sky(coord_gals_z, coord_neighbors,
aper_size * 2 / 3600 * u.deg)
gal1_list = arr[0][arr[2].value > 2.5 / 3600]
gal2_list = arr[1][arr[2].value > 2.5 / 3600]
no_pairs = len(gal1_list)
no_pairs_all = len(arr_all[0])
if no_pairs > 0:
print('no of objects', len(cat_neighbors_z), 'no. of close pairs',
no_pairs, 'no. of pairs ', no_pairs_all)
else:
continue
# calculate deltaz
for k in range(len(gal1_list)):
if gal1_list[k] < gal2_list[k]: # only unique pairs
def run_on_single_catalog(self, catalog_instance, catalog_name,
output_dir):
# pylint: disable=no-member
# Try to read cosmology from catalog, otherwise defualts to WMAP7
try:
cosmo = catalog_instance.cosmology
except AttributeError:
cosmo = WMAP7
# Create interpolation tables for efficient computation of sigma crit
z = np.linspace(0, self.zmax, self.zmax * 100)
d1 = cosmo.angular_diameter_distance(z) # in Mpc
angular_diameter_distance = interp1d(z, d1, kind='quadratic')
d2 = cosmo.comoving_transverse_distance(z) # in Mpc
comoving_transverse_distance = interp1d(z, d2, kind='quadratic')
res = catalog_instance.get_quantities([
'redshift_true', 'ra', 'dec', 'shear_1', 'shear_2', 'convergence',
'mag_true_i_sdss', 'mag_true_z_sdss', 'mag_true_g_sdss',
'mag_true_r_sdss'
])
# Compute mask for lowz sample
# These cuts are defined in section 3 of https://arxiv.org/pdf/1509.06529.pdf
# and summarised here: http://www.sdss.org/dr14/algorithms/boss_galaxy_ts/#TheBOSSLOWZGalaxySample
# Definition of auxiliary colors:
cperp = (res['mag_true_r_sdss'] - res['mag_true_i_sdss']) - (
res['mag_true_g_sdss'] - res['mag_true_r_sdss']) / 4.0 - 0.18
cpar = 0.7 * (
res['mag_true_g_sdss'] - res['mag_true_r_sdss']) + 1.2 * (
(res['mag_true_r_sdss'] - res['mag_true_i_sdss']) - 0.18)
# LOWZ selection cuts:
mask_lowz = np.abs(cperp) < 0.2 # color boundaries
mask_lowz &= res['mag_true_r_sdss'] < (13.5 + cpar / 0.3
) # sliding magnitude cut
mask_lowz &= (res['mag_true_r_sdss'] > 16) & (res['mag_true_r_sdss'] <
19.6)
# Counting the number density of LOWZ galaxies
nlens = len(np.where(mask_lowz)[0]) / catalog_instance.sky_area
with open(os.path.join(output_dir, 'galaxy_density.dat'), 'a') as f:
f.write('{} \n'.format(nlens))
# Additional redshift cuts used in Singh et al. (2015)
mask_lowz &= (res['redshift_true'] > 0.16) & (res['redshift_true'] <
0.36)
# Computing mask for source sample, this only serves to keep the number
# of neighbours manageable
mask_source = res['redshift_true'] > self.zcut_background
inds = np.where(mask_source)[0]
if len(inds) > int(self.max_background_galaxies):
mask_source[inds[np.random.choice(
len(inds),
size=len(inds) - int(self.max_background_galaxies),
replace=False)]] = False
# Create astropy coordinate objects
coords = SkyCoord(ra=res['ra'] * u.degree, dec=res['dec'] * u.degree)
coords_l = coords[mask_lowz]
coords_s = coords[mask_source]
# Search for neighbours
idx1, idx2, sep2d, _ = search_around_sky(coords_l, coords_s,
2. * u.deg)
# Computing sigma crit for each pair
zl = res['redshift_true'][mask_lowz][idx1]
zs = res['redshift_true'][mask_source][idx2]
# Warning: this assumes a flat universe
# See http://docs.astropy.org/en/v0.3/_modules/astropy/cosmology/core.html#FLRW.angular_diameter_distance_z1z2
dm1 = comoving_transverse_distance(zl)
dm2 = comoving_transverse_distance(zs)
angular_diameter_distance_z1z2 = u.Quantity((dm2 - dm1) / (1. + zs),
u.Mpc)
sigcrit = cst.c**2 / (4.*np.pi*cst.G) * angular_diameter_distance(zs) / \
((1. + zl)**2. * angular_diameter_distance_z1z2 * angular_diameter_distance(zl))
# NOTE: the validation data is in comoving coordinates, the next few
# lines take care of proper unit conversions
# Apply unit conversion to obtain sigma crit in h Msol /pc^2 (comoving)
cms = u.Msun / u.pc**2
sigcrit = sigcrit * (u.kg / (u.Mpc * u.m)).to(cms) / cosmo.h
# Computing the projected separation for each pairs, in Mpc/h (comoving)
r = sep2d.rad * angular_diameter_distance(zl) * (1. + zl) * cosmo.h
# Computing the tangential shear
thetac = np.arctan2(
(coords_s[idx2].dec.rad - coords_l[idx1].dec.rad) / np.cos(
(coords_s[idx2].dec.rad + coords_l[idx1].dec.rad) / 2.0),
coords_s[idx2].ra.rad - coords_l[idx1].ra.rad)
gammat = -(res['shear_1'][mask_source][idx2] * np.cos(2 * thetac) -
res['shear_2'][mask_source][idx2] * np.sin(2 * thetac))
# Binning the tangential shear
bins = np.logspace(np.log10(0.05), 1, 17, endpoint=True)
counts = np.histogram(r, bins=bins)[0]
gt, b = np.histogram(r, bins=bins, weights=gammat * sigcrit)
rp = 0.5 * (b[1:] + b[:-1])
# Outputs the number of background galaxies in each bins and checks that
# that number is sufficient.
with open(os.path.join(output_dir, 'galaxy_count.dat'), 'a') as f:
f.write('{} {} {}\n'.format(rp, counts, gt))
if counts.min() < self.min_count_per_bin:
return TestResult(
score=np.nan,
passed=False,
summary="Not enough background sources to compute delta sigma")
gt = gt / counts
fig = plt.figure()
ax = plt.subplot(111)
plt.loglog(rp, gt, label='LOWZ-like sample from ' + catalog_name)
plt.errorbar(self.validation_data[:, 0],
self.validation_data[:, 1],
yerr=self.validation_data[:, 2],
label='SDSS LOWZ from Singh et al. (2015)')
plt.title(
'Number density {}/deg$^2$ vs 57/deg$^2$ for LOWZ'.format(nlens))
ax.set_xlabel('$r_p$ [Mpc/h]')
ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$')
ax.legend()
ax.set_xlim(0.05, 10)
ax.set_ylim(0.5, 100)
fig.savefig(
os.path.join(output_dir,
'delta_sigma_{}.png'.format(catalog_name)))
plt.close(fig)
return TestResult(inspect_only=True)
def filter_cmb(
path="/home/chema/Escritorio/datos/LFI_SkyMap_030-BPassCorrected_1024_R3.00_full.fits",
f=30,
fwhm=32.65,
cut=4,
print_sources=False):
star_time = time.time()
# Loading of the FITS file with the image of the CMB and change the units to Jy. Specify the path to the file,
# the frequency and the fwhm of the instrument.
f, fwhm = f * u.GHz, fwhm * u.arcmin # Input data
# The first element of the array is the one with the T map.
object_map = Fitsmap.from_file(path, freq=f, fwhm=fwhm)[0]
print("--- Data loaded ---")
# Changing data from K to Jy
object_map.to_Jy()
# --- Parameters that characterize the patches that will be used to cover the CMB map on
# which we will use the filter ---
patch_size = 256
# --- Computing the center of the patches to cover all the sphere ---
nside = 8
npix = hp.nside2npix(nside)
centers = [hp.pix2vec(nside, i) for i in range(npix)]
# List with the position vectors of the npix of side = 8 that will be the center of the patches.
lon, lat = hp.vec2ang(np.array(centers),
lonlat=True) # Center vector to lon and lat in deg.
npatches = len(centers)
# --- Filtering every patch and getting all the candidates of compact sources, gathering all in a table ---
peaks = []
for i in trange(npatches):
coord = SkyCoord(l=lon[i], b=lat[i], frame='galactic', unit=u.deg)
patch = object_map.patch(coord, npix=patch_size)
filtered_patch = patch.matched(fwhm=object_map.fwhm)
std = filtered_patch.std()
threshold = cut * std
indices = peak_local_max(filtered_patch.datos,
min_distance=2,
threshold_abs=threshold,
exclude_border=True,
indices=True)
for x, y in indices:
coordinate = filtered_patch.pixel_coordinate(x, y)
# Estimation of the flux error
s = (object_map.fwhm / filtered_patch.pixsize).si.value
w = np.arange(0, patch_size, 1, float)
v = w[:, np.newaxis]
r = np.sqrt((w - x)**2 + (v - y)**2)
m = (r >= 3 * s) & (r <= 5 * s)
peak = {
'Longitude': coordinate.l.value,
'Latitude': coordinate.b.value,
'Flux': filtered_patch.datos[x, y],
'Flux_error': filtered_patch.datos[m].std(),
'SNR': filtered_patch.datos[x, y] / std
}
peaks.append(peak)
print("--- Map filtered ---")
table = Table(peaks)
coordinates = SkyCoord(
l=np.array(table['Longitude'].tolist()),
b=np.array(table['Latitude'].tolist()),
frame='galactic',
unit=u.deg
) # Coordinates of each peak in the galactic system (in angles).
# --- Searching for repetitions in our catalogue ---
resolution_max = object_map.fwhm.to(u.arcmin)
ind1, ind2, dist, d3d = search_around_sky(coordinates, coordinates,
3 / 4 * resolution_max)
correspondence_table = Table()
correspondence_table['Index 1'] = ind1
correspondence_table['Index 2'] = ind2
correspondence_table['Separation'] = dist
final_table = Table()
# --- Resolution of coincidences ---
while len(correspondence_table) > len(final_table):
skipped = []
final_table = Table()
for i in trange(len(table)):
if i == 0:
final_table = Table(rows=table[i],
names=('Longitude', 'Latitude', 'Flux',
'Flux_error', 'SNR'))
if i in skipped:
continue
x = correspondence_table[correspondence_table['Index 1'] == i]
if len(x) == 1:
if i == 0:
continue
else:
final_table.add_row([
table['Longitude'][i], table['Latitude'][i],
table['Flux'][i], table['Flux_error'][i],
table['SNR'][i]
])
else:
skipped = np.append(skipped, np.array(x['Index 2'].tolist()))
coord = SkyCoord(l=table['Longitude'].tolist()[i],
b=table['Latitude'].tolist()[i],
frame='galactic',
unit=u.deg)
patch = object_map.patch(coord,
npix=int(patch_size / 2),
deltatheta_deg=14.658 / 2)
filtered_patch = patch.matched(fwhm=object_map.fwhm)
std = filtered_patch.std()
threshold = cut * std
indices = peak_local_max(filtered_patch.datos,
min_distance=2,
threshold_abs=threshold,
exclude_border=True,
indices=True)
for x, y in indices:
s = (object_map.fwhm / filtered_patch.pixsize).si.value
if (x >= patch_size/4 - 3/4*s) & (x <= patch_size/4 + 3/4*s) & \
(y >= patch_size/4 - 3/4*s) & (y <= patch_size/4 + 3/4*s):
coordinate = patch.pixel_coordinate(x, y)
# Estimation of the flux error
w = np.arange(0, int(patch_size / 2), 1, float)
v = w[:, np.newaxis]
r = np.sqrt((w - x)**2 + (v - y)**2)
m = (r >= 3 * s) & (r <= 5 * s)
peak = {
'Longitude': coordinate.l.value,
'Latitude': coordinate.b.value,
'Flux': filtered_patch.datos[x, y],
'Flux_error': filtered_patch.datos[m].std(),
'SNR': filtered_patch.datos[x, y] / std
}
final_table.add_row([
peak['Longitude'], peak['Latitude'], peak['Flux'],
peak['Flux_error'], peak['SNR']
])
if i == 0:
final_table.remove_row(0)
coordinates = SkyCoord(l=np.array(final_table['Longitude'].tolist()),
b=np.array(final_table['Latitude'].tolist()),
frame='galactic',
unit=u.deg)
ind1, ind2, dist, d3d = search_around_sky(coordinates, coordinates,
3 / 4 * resolution_max)
correspondence_table = Table()
correspondence_table['Index 1'] = ind1
correspondence_table['Index 2'] = ind2
correspondence_table['Separation'] = dist
table = final_table
print("--- Duplicated sources removed ---")
# Saving the table with the catalogue.
gal_coords = SkyCoord(l=np.array(table['Longitude'].tolist()),
b=np.array(table['Latitude'].tolist()),
frame='galactic',
unit=u.deg)
eq_coords = gal_coords.icrs
table.add_column(col=eq_coords.ra, index=2, name='RA')
table.add_column(col=eq_coords.dec, index=2, name='DEC')
table['Longitude'].unit, table['Latitude'].unit = 'deg', 'deg'
table['Flux'].unit, table['Flux_error'].unit = 'Jy', 'Jy'
table['RA'].unit, table['DEC'].unit = 'deg', 'deg'
table.write('catalogue_' + str(f.value) + 'GHz.fits', overwrite=True)
print("--- Data saved ---")
# Saving a picture with the localization of all the sources.
if print_sources:
hp.projscatter(table['Longitude'].data.data,
table['Latitude'].data.data,
lonlat=True,
coord='G',
color='r',
marker='.')
hp.graticule()
hp.projscatter(table['Longitude'].data.data,
table['Latitude'].data.data,
lonlat=True,
coord='G',
color='r',
marker='.')
plt.savefig('Detected_Sources_' + str(f) + '_SNR' + str(cut) + '.png')
print(str(len(table)) + ' compact sources detected.')
print('Execution time: ' +
str(time.strftime("%M:%S", time.gmtime(time.time() - star_time))) +
' (min:s)')
def calculate_sim_images(simfolder, vis=None, baseimage=None, baseimage_file=None, repeat=10,
basename=None, savefile=None, fov_scale=1.5, second_check=False,
detection_threshold=2.5, aperture_size=None, aperture_scale=6.0,
plot=False, snr_mode='peak', debug=False, overwrite=True,
method='sep',
snr=[1,20], seplimit_scale=0.5, seplimit_arcsec=None, mask=None, **kwargs):
"""simulation the completeness of source finding algorithm
mode:
peak: snr is the peak value
integrated: snr is the integrated value
seplimit: in arcsec
The second_check is set to false to get the completeness across the whole SNR
"""
if not isinstance(baseimage, FitsImage):
if os.path.isfile(baseimage_file):
baseimage = FitsImage(baseimage_file)
else:
raise ValueError("No valid baseimage or file has been provided!")
# recalibrate the std based one the masked image
baseimage.imagemask = baseimage.mask_image(mask=baseimage.find_structure(sigma=3.0))
baseimage.imstat()
# known_sources = source_finder(baseimage, fov_scale=fov_scale)
if basename is None:
basename = baseimage.name
# define the statistical variables
boosting_table = Table(names=['snr_peak_input', 'snr_peak_found', 'flux_input','flux_aperture','flux_gaussian'],
dtype=['f8', 'f8', 'f8', 'f8', 'f8'])
comp_table = Table(names=['snr_peak', 'is_recovered'], dtype=['f8', 'int'])
fake_table = Table(names=['snr_peak', 'is_fake'], dtype=['f8','int'])
for run in np.arange(repeat):
if debug:
print("calculating run: {}".format(run))
# try:
if True:
simimage_imagefile = "{basename}.run{run}.image.fits".format(basename=basename, run=run)
simimage_sourcefile = "{basename}.run{run}.txt".format(basename=basename, run=run)
simimage_fullpath = os.path.join(simfolder, simimage_imagefile)
simimage_sourcefile_fullpath = os.path.join(simfolder, simimage_sourcefile)
# print("simulated image:", simimage_fullpath)
# print("simulated sources:", simimage_sourcefile_fullpath)
##### for new simulations
# sources_input = Table.read(simimage_sourcefile_fullpath, format='ascii')
# sources_input_coords = SkyCoord(ra=sources_input['ra[deg]']*u.deg,
# dec=sources_input['dec[deg]']*u.deg)
# sources_input_flux = sources_input['flux[Jy]'] #convert to Jy
##### for old simulations
sources_input = Table.read(simimage_sourcefile_fullpath, format='ascii')
sources_input_coords = SkyCoord(ra=sources_input['ra[arcsec]']*u.arcsec,
dec=sources_input['dec[arcsec]']*u.arcsec)
sources_input_flux = sources_input['flux[mJy]']/1000 #convert to Jy
simimage = FitsImage(simimage_fullpath)
sources_found = source_finder(simimage, detection_threshold=detection_threshold,
method=method, mask=mask)
sources_found_coords = SkyCoord(ra=sources_found['ra']*u.deg,
dec=sources_found['dec']*u.deg)
if seplimit_scale is not None:
seplimit = seplimit_scale * simimage.bmaj*3600 * u.arcsec
if seplimit_arcsec is not None:
seplimit = seplimit_arcsec * u.arcsec
if len(sources_input) > 0:
if len(sources_found) > 0:
idx_input, idx_found, d2d, d3d = search_around_sky(sources_input_coords,
sources_found_coords,
seplimit)
idx_input_comp = np.array(list(set(range(len(sources_input))) - set(idx_input)),
dtype=int)
idx_found_comp = np.array(list(set(range(len(sources_found))) - set(idx_found)),
dtype=int)
# calculate the flux boosting
snr_found_boosting = sources_found['peak_flux'][idx_found] / baseimage.std
flux_aperture, flux_aperture_err = measure_flux(simimage,
detections=sources_found[idx_found],
aperture_size=aperture_size,
aperture_scale=aperture_scale,
minimal_aperture_size=simimage.bmaj*2*3600,
method='single-aperture')
flux_gaussian, flux_gaussian_err = measure_flux(simimage,
detections=sources_found[idx_found],
method='gaussian')
flux_input = sources_input_flux[idx_input]
snr_input_boosting = sources_input_flux[idx_input] / baseimage.std
boosting_single = Table([snr_found_boosting, snr_input_boosting, snr_found_boosting, flux_input, flux_aperture, flux_gaussian],
names=['snr_peak','snr_peak_input','snr_peak_found','flux_input','flux_aperture','flux_gaussian'])
boosting_table = vstack([boosting_table, boosting_single])
# calculate the completeness
snr_input_completeness = sources_input_flux/baseimage.std
is_recovered = np.zeros_like(snr_input_completeness).astype(int)
is_recovered[idx_input] = 1
is_recovered[idx_input_comp] = 0
comp_single = Table([snr_input_completeness, is_recovered], names=['snr_peak', 'is_recovered'])
comp_table = vstack([comp_table, comp_single])
# calculate the false detection
snr_found_fakeness = sources_found['peak_flux']/baseimage.std
is_fake = np.zeros_like(snr_found_fakeness).astype(int)
is_fake[idx_found] = 0
is_fake[idx_found_comp] = 1
fake_single = Table([snr_found_fakeness, is_fake], names=['snr_peak', 'is_fake'])
fake_table = vstack([fake_table, fake_single])
# except:
# print('Failed in run {}'.format(run))
# continue
if savefile:
boosting_table.write(savefile+'_boosting.dat', format='ascii', overwrite=overwrite)
comp_table.write(savefile+'_completeness.dat', format='ascii', overwrite=overwrite)
fake_table.write(savefile+'_fake.dat', format='ascii', overwrite=overwrite)
return boosting_table
def add_to_hdf(hdf, Z_MIN = 0.1, Z_MAX = 7.1, MATCH_TOL = 2.0*u.arcsec):
"""Generate Myers + SDSS_BOSS QSO catalog from Myers and DR12 files
This routine reads in the SDSS/BOSS specObj and PhotPosPlate files
and pulls out all QSOs. This is then matched amd merged with
the Myers QSO compilation with duplicates removed. It adds this
catalog to the 'quasars' field of the hdf5 object hdf
Requires
that that environment varialble RAW_IGMSPEC be set to the top
directory where the SDSS/BOSS files and Myers files live.
Parameters
----------
hdf : hdf5 object
database to hold QSO catalog
Z_MIN : float, optional [default Z_MIN = 0.1]
minimum QSO redshift applied to the catalog
Z_MAX : float. optimal [default Z_MAX = 7.1]
maximum QSO redshift applied to the catalo
MATCH_TOL : quantity, optional [default 2.0*u.arcsec]
matching radius between Myers and SDSS/BOSS catalogs
Returns
-------
None :
None
Examples
--------
>>> add_to_hdf(hdf)
None
Notes
-----
In the 'quasars' table added to the hdf5 object hdf, tags with the SDSS_BOSS_ prefix are
the SDSS/BOSS tags, and tags with MYERS_ are from the Myers catalog. We also add the following
tags
'SDSS_BOSS_MYERS_FLAG' = set to either SDSS_BOSS_MYERS, SDSS_BOSS_ONLY, or MYERS_ONLY
'RA', 'DEC' = our best knowledge of the coordinates (PLUG_RA, PLUG_DEC for SDSS/BOSS, otherwise Myers)
'SOURCEBIT' = Myers catalog sourcebits, with a new 2**19 sourcebit for objects in SDSS_BOSS_ONLY objects
'ZEM' = Our best knwoledge of the redshift ( either MYERS_ZEM or SDSS/BOSS redshift for SDSS_BOSS_ONLY objects
'ZEM_SOURCE' = The source of the redshift following the Myers classification with an additional SDSS_BOSS_ONLY
"""
import json
from astropy.table import Column, hstack, vstack
from astropy.coordinates import SkyCoord, search_around_sky
## SDSS/BOSS data stuff
specfile = os.getenv('RAW_IGMSPEC') + '/SDSS_BOSS/specObj-dr12_trim.fits'
spec = Table.read(specfile)
# Read in select columns from DR12 photometry. This and the file above are aligned
posfile = os.getenv('RAW_IGMSPEC') + '/SDSS_BOSS/photoPosPlate-dr12_trim.fits'
phot = Table.read(posfile)
# Trim to QSO, Specprimary, spec2d called it a QSO, redshift flag cuts, sanity check on coords
itrim = (spec['SPECPRIMARY'] == 1) & \
[('QSO' in q) for q in spec['CLASS']] & \
(spec['ZWARNING'] < 5) & \
(spec['PLUG_RA'] >= 0.0) & (spec['PLUG_RA'] <= 360.0) & \
(np.abs(spec['PLUG_DEC']) <= 90.0)
spec = spec[itrim]
phot = phot[itrim]
sdss_boss1 = hstack([spec, phot], join_type='exact')
# Add SDSS prefix to all SDSS tags
for key in sdss_boss1.keys():
sdss_boss1.rename_column(key, 'SDSS_BOSS_' + key)
# Read in the Myers file, match it to Myers sweeps photometry
# Myers master QSO catalog
ADM_file = os.getenv('RAW_IGMSPEC') + '/Myers/GTR-ADM-QSO-master-wvcv.fits.gz'
ADM_qso = Table.read(ADM_file)
head1 = fits.open(ADM_file)[1].header
DATE = head1['DATE']
# Photometry for Myers QSO catalog. This file is not aligned with the catalog file, i.e. it is a
# superset that includes the catalog file. For that reason we need to match and tack on photometry
ADM_sweep_file = os.getenv('RAW_IGMSPEC') + '/Myers/GTR-ADM-QSO-master-sweeps-Feb5-2016.fits'
ADM_sweep = Table.read(ADM_sweep_file)
c_qso = SkyCoord(ra=ADM_qso['RA'], dec=ADM_qso['DEC'],unit ='deg')
c_swp = SkyCoord(ra=ADM_sweep['RA'], dec=ADM_sweep['DEC'], unit='deg')
## Create an aligned Table for matching photometry from sweeps
nqso = len(ADM_qso)
qso_phot = Table(np.repeat(np.zeros_like(ADM_sweep[0]), nqso))
# Rename the RA and DEC
qso_phot.rename_column('RA', 'RA_sweep')
qso_phot.rename_column('DEC', 'DEC_sweep')
# Cull out the keys which already exist in the ADM_qso Table (except
# for the RA and DEC, which we renamed)
dupe_keys = list(set(ADM_qso.keys()) & set(qso_phot.keys()))
qso_phot.remove_columns(dupe_keys)
# Match the Myers catalog to the Myers sweeps
idx, d2d, d3d = c_qso.match_to_catalog_sky(c_swp)
# Currently using 1.0" for matching, as for the SDSS objects, these will mostly be the exact
# same coordinates.
itrim = (d2d <= 1.0 * u.arcsec)
qso_phot[:][itrim] = ADM_sweep[:][idx[itrim]]
ADM_qso = hstack([ADM_qso, qso_phot], join_type='exact')
# Trim to only spectroscopic objects
ispec = spectro_myers(ADM_qso)
ADM_qso = ADM_qso[ispec]
# assign best redshifts to ZEM tag
zbest_myers(ADM_qso)
# Add MYERS prefix to all MYERS tags
for key in ADM_qso.keys():
ADM_qso.rename_column(key, 'MYERS_' + key)
# Now we meatch the SDSS/BOSS and Myers catalogs to create one master QSO catalog
#
# There are three groups of objects, 1) SDSS-MYERS match, 2) SDSS only, 3) Myers only.
# Deal with each in turn.
# 1) SDSS-MYERS match. Add Myers tags to the SDSS structure
c_sdss = SkyCoord(ra=sdss_boss1['SDSS_BOSS_PLUG_RA'], dec=sdss_boss1['SDSS_BOSS_PLUG_DEC'], unit='deg')
c_myers = SkyCoord(ra=ADM_qso['MYERS_RA'], dec=ADM_qso['MYERS_DEC'], unit='deg')
isdss, imyers, d2d, _ = search_around_sky(c_sdss, c_myers, MATCH_TOL)
sdss_myers = hstack([sdss_boss1[isdss], ADM_qso[imyers]], join_type='exact')
sdss_myers['SDSS_BOSS_MYERS_FLAG'] = 'SDSS_BOSS_MYERS'
sdss_myers['RA'] = sdss_myers['SDSS_BOSS_PLUG_RA'] # SDSS/BOSS Plug coords most accurate
sdss_myers['DEC'] = sdss_myers['SDSS_BOSS_PLUG_DEC']
sdss_myers['SOURCEBIT'] = sdss_myers['MYERS_SOURCEBIT']
sdss_myers['ZEM'] = sdss_myers['MYERS_ZEM']
sdss_myers['ZEM_SOURCE'] = sdss_myers['MYERS_ZEM_SOURCE']
# 2) SDSS only
# Find the SDSS objects that have no match in the Myers catalog
inomatch = np.ones(len(c_sdss), dtype=bool)
inomatch[isdss] = False
sdss_only = sdss_boss1[inomatch]
sdss_only['SDSS_BOSS_MYERS_FLAG'] = 'SDSS_BOSS_ONLY'
sdss_only['RA'] = sdss_only['SDSS_BOSS_PLUG_RA']
sdss_only['DEC'] = sdss_only['SDSS_BOSS_PLUG_DEC']
sdss_only['SOURCEBIT'] = 2 ** 19 # New source bit for SDSS only objects
sdss_only['ZEM'] = sdss_only['SDSS_BOSS_Z']
sdss_only['ZEM_SOURCE'] = 'SDSS_BOSS_ONLY'
# 3) Myers only
# Find the Myers objects that have no match in SDSS/BOSS
inomatch = np.ones(len(c_myers), dtype=bool)
inomatch[imyers] = False
myers_only = ADM_qso[inomatch]
myers_only['SDSS_BOSS_MYERS_FLAG'] = 'MYERS_ONLY'
myers_only['RA'] = myers_only['MYERS_RA']
myers_only['DEC'] = myers_only['MYERS_DEC']
myers_only['SOURCEBIT'] = myers_only['MYERS_SOURCEBIT']
myers_only['ZEM'] = myers_only['MYERS_ZEM']
myers_only['ZEM_SOURCE'] = myers_only['MYERS_ZEM_SOURCE']
sdss_myers_out = vstack([sdss_myers, sdss_only, myers_only])
# Cut down
ztrim = (sdss_myers_out['ZEM'] >= Z_MIN) & (sdss_myers_out['ZEM'] <= Z_MAX)
coordtrim = (sdss_myers_out['RA'] >= 0.0) & (sdss_myers_out['RA'] <= 360.0) & (np.abs(
sdss_myers_out['DEC']) <= 90.0)
keep = ztrim & coordtrim
sdss_myers_out = sdss_myers_out[keep]
# Clean out unicode
sbu.clean_table_for_hdf(sdss_myers_out)
hdf['quasars'] = sdss_myers_out
hdf['quasars'].attrs['MYERS_DATE'] = DATE
# Myers dict
mdict = myers_dict()
hdf['quasars'].attrs['MYERS_DICT'] = json.dumps(ltu.jsonify(mdict))
return None
def check_splus(ra, dec, tilesep_deg, catsep_arcsec):
print()
obj = create_sky_object(ra, dec)
print("Input location (RA,DEC): ", obj.ra.value[0], 'deg',
obj.dec.value[0], 'deg')
# read SPLUS pointings file
splus_tiles_file = 'data/all_pointings.csv'
tiles = Table.read(splus_tiles_file, format="csv")
tiles_coords = SkyCoord(ra=tiles['RA'],
dec=tiles['DEC'],
unit=(u.hourangle, u.degree))
# read IDR2 pointings file
idr2_tiles_file = 'data/idr2_pointings.csv'
idr2_tiles = Table.read(idr2_tiles_file, format="csv")
sky_radius_deg = tilesep_deg * u.degree
print("Searching all S-PLUS pointings...")
idx_data1, idx_data2, d2d, d3d = search_around_sky(obj, tiles_coords,
sky_radius_deg)
if len(d2d) > 0:
for i in range(0, len(d2d)):
location_in_idr2 = '(NOT IN DR2)'
sel = (idr2_tiles['FIELD'] == tiles['NAME'][idx_data2[i]])
nfound = np.sum(sel)
if nfound > 0:
location_in_idr2 = '(IDR2)'
print(
"Your location is within{0:4.1f} degrees of the center of pointings:"
.format(tilesep_deg))
print("{0:s} {1:s}/{2:s} (separation:{3:5.2f})".format(
location_in_idr2, tiles['PID'][idx_data2[i]],
tiles['NAME'][idx_data2[i]], d2d[i].value))
else:
print("Nothing found.")
print()
print("Searching all iDR2 objects...")
infile = 'Catalogs/merged_id.fits'
poscat = Table.read(infile, format='fits')
obscoords = SkyCoord(ra=poscat['RA'],
dec=poscat['Dec'],
unit=(u.degree, u.degree))
sky_radius_deg = (catsep_arcsec / 3600.) * u.degree
idx_data1, idx_data2, d2d, d3d = search_around_sky(obj, obscoords,
sky_radius_deg)
sor = np.argsort(d2d)
idx_data1 = idx_data1[sor]
idx_data2 = idx_data2[sor]
d2d = d2d[sor]
d3d = d3d[sor]
if len(d2d) > 0:
for i in range(0, len(d2d)):
print(
"Object {0:s} in area/tile {1:7s}/{2:12s} with separation: {3:5.2f} arcsec)"
.format(str(poscat['ID'][idx_data2[i]]),
poscat['SURVEY'][idx_data2[i]],
poscat['FIELD'][idx_data2[i]], d2d[i].value * 3600.))
else:
print("Nothing found.")
return
def add_spectra(base, specs):
if 'coord' in specs.colnames:
del specs['coord']
specs.sort(['ZQUALITY', 'SPEC_Z'])
specs.reverse()
specs = add_skycoord(specs)
specs_idx, base_idx, sep, _ = search_around_sky(
specs['coord'], base['coord'], 20.0 * astropy.units.arcsec) # pylint: disable=E1101
sep = sep.arcsec
if (np.ediff1d(specs_idx) < 0).any():
sorter = specs_idx.argsort()
specs_idx = specs_idx[sorter]
base_idx = base_idx[sorter]
sep = sep[sorter]
del sorter
base['index'] = np.arange(len(base))
base['no_spec_yet'] = True
specs['matched_idx'] = -1
specs_idx_edges = np.flatnonzero(
np.hstack(([1], np.ediff1d(specs_idx), [1])))
for i, j in zip(specs_idx_edges[:-1], specs_idx_edges[1:]):
spec_idx_this = specs_idx[i]
possible_match = base['REMOVE', 'is_galaxy', 'r_mag', 'radius',
'no_spec_yet', 'index'][base_idx[i:j]]
possible_match['sep'] = sep[i:j]
possible_match[
'sep_norm'] = possible_match['sep'] / possible_match['radius']
if Query('sep < 1', ~Query('no_spec_yet')).count(possible_match) > 0:
specs['matched_idx'][spec_idx_this] = -2 # duplicated specs
possible_match = possible_match[possible_match['no_spec_yet']]
if not len(possible_match):
continue
larger_search_r = build._get_spec_search_radius(
specs['SPEC_Z'][spec_idx_this])
for q, sorter in (
(Query('REMOVE == 0', ~Query('is_galaxy'), 'sep < 0.5'), 'sep'),
(Query('REMOVE == 0', 'is_galaxy', 'sep_norm < 1',
'sep < {:g}'.format(larger_search_r)), 'r_mag'),
(Query('REMOVE == 0', 'is_galaxy', 'sep < 3'), 'r_mag'),
(Query('REMOVE == 0', ~Query('is_galaxy'), 'sep < 3'), 'sep'),
(Query('REMOVE > 0', ~Query('is_galaxy'), 'sep < 0.5'), 'sep'),
(Query('REMOVE > 0', 'is_galaxy', 'sep_norm < 1',
'sep < {:g}'.format(larger_search_r)), 'r_mag'),
(Query('REMOVE > 0', 'is_galaxy', 'sep < 3'), 'r_mag'),
(Query('REMOVE > 0', ~Query('is_galaxy'), 'sep < 3'), 'sep'),
):
mask = q.mask(possible_match)
if mask.any():
possible_match_this = possible_match[mask]
matched_base_idx = possible_match_this['index'][
possible_match_this[sorter].argmin()]
specs['matched_idx'][spec_idx_this] = matched_base_idx
base['no_spec_yet'][matched_base_idx] = False
break
del base['index'], base['no_spec_yet'], specs[
'coord'], specs_idx, base_idx, sep, specs_idx_edges
specs.sort('matched_idx')
start_idx = np.flatnonzero(specs['matched_idx'] > -1)[0]
for col in tuple(SPECS_COLUMNS) + ('SPEC_REPEAT', 'OBJ_NSAID'):
col_base = (col + '_spec') if col in ('RA', 'DEC') else col
base[col_base][specs['matched_idx']
[start_idx:]] = specs[col][start_idx:]
specs_warn = specs[np.flatnonzero(specs['matched_idx'] > -2)[0]:start_idx]
specs_warn = specs_warn[specs_warn['SPEC_REPEAT'] != 'GAMA']
if len(specs_warn) > 100:
logging.warning(
'More than 100 spec objects have no match... something is very wrong!'
)
else:
for spec in specs_warn:
logging.warning(
'No photo obj matched to {} spec obj {} ({}, {})'.format(
spec['TELNAME'], spec['SPECOBJID'], spec['RA'],
spec['DEC']))
del specs['matched_idx']
return base
def run_bayes(frb_tbl_file,
galaxy_cat_file,
prior,
outfile,
frb_ee,
galaxy_coords=None,
max_radius=20.,
mag_lim=None,
ncpu=15,
multi=True):
"""
Run the Bayes analysis
Args:
frb_tbl_file:
galaxy_cat_file:
prior:
outfile:
frb_ee:
galaxy_coords:
max_radius:
Returns:
"""
# Defs
sb_defs = cosmos_defs()
# Load FRBs
print("Loading...")
sb_frbs = pandas.read_csv(frb_tbl_file, index_col=0)
nFRB = len(sb_frbs)
# frb_ee
if isinstance(frb_ee, dict):
frb_ee_list = [frb_ee] * nFRB
elif isinstance(frb_ee, str):
frb_ee_list = []
if frb_ee == 'loc_sig':
for kk in range(nFRB):
frb_ee_list.append({
'a': sb_frbs.iloc[kk].loc_sig,
'b': sb_frbs.iloc[kk].loc_sig,
'theta': 0.
})
else:
raise IOError("Bad frb_ee")
# Load Galaxies
sb_galaxies = pandas.read_feather(galaxy_cat_file)
# Cut
sb_galaxies = sb_galaxies[np.isfinite(sb_galaxies.a_image)
& np.isfinite(sb_galaxies.mag_best)]
# Extra bits
sb_galaxies['half_light'] = sb_galaxies.a_image * sb_defs['plate_scale']
sb_galaxies[sb_defs['filter']] = sb_galaxies.mag_best
# Cut on mag?
if mag_lim is not None:
keep = sb_galaxies.mag_best < mag_lim
sb_galaxies = sb_galaxies[keep]
# Coordinates (this is slow)
print("Building coords...")
frb_coords = SkyCoord(ra=sb_frbs.frb_ra, dec=sb_frbs.frb_dec, unit='deg')
if galaxy_coords is None:
galaxy_coords = SkyCoord(ra=sb_galaxies.ra,
dec=sb_galaxies.dec,
unit='deg')
# Find the candidates
print("Finding candidates...")
idx1, idx2, sep2d, _ = search_around_sky(galaxy_coords, frb_coords,
15 * units.arcsec)
# Slice the tables
print("Slicing...")
list_candidates = []
#nFRB = 2000
for kk in range(nFRB):
if (kk % 1000) == 0:
print('kk: ', kk)
in_idx2 = np.where(idx2 == kk)[0]
gd_gal = idx1[in_idx2]
close_galaxies = sb_galaxies.iloc[gd_gal][
[ #'mag_best', #'a_image', #'kron_radius',
#'b_image', #'ra', 'dec', 'class_star',
'half_light',
sb_defs['filter'],
'ra',
'dec'
]]
# Extras
close_galaxies['separation'] = sep2d[gd_gal].to('arcsec').value
close_galaxies['coords'] = galaxy_coords[gd_gal]
#
list_candidates.append(close_galaxies)
# Loop me!
print('Starting to loop!')
# Build
if multi:
pool = multiprocessing.Pool(processes=ncpu)
results = [
pool.apply_async(
main_calc,
args=(idx_FRB, frb_coords[idx_FRB], frb_ee_list[idx_FRB],
list_candidates[idx_FRB], sb_defs, prior, max_radius))
for idx_FRB in range(nFRB)
]
output = [p.get() for p in results]
idx = [item[1] for item in output]
all_tbls = np.array([item[0] for item in output], dtype=object)
all_tbls = all_tbls[idx]
# Expunge the None's
gd_tbl = np.array(
[False if item is None else True for item in all_tbls])
gd_idx = np.arange(all_tbls.size)[gd_tbl]
all_tbls = all_tbls[gd_tbl]
for kk in range(all_tbls.size):
all_tbls[kk]['iFRB'] = gd_idx[kk]
else:
embed(header='broken')
all_tbls = []
for idx_FRB in range(nFRB):
sv_tbl = main_calc(frb_coords[idx_FRB], sb_galaxies, galaxy_coords,
FRB, frbA, sb_defs, prior)
sv_tbl['iFRB'] = idx_FRB
all_tbls.append(sv_tbl)
# Finish
final_tbl = pandas.concat(all_tbls)
final_tbl.to_csv(outfile)
print("Wrote: {}".format(outfile))
cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sfProb_nuvrk'] > 0.5]
cat_neighbors = cat_neighbors[cat_neighbors['sfProb_nuvrk'] > 0.5]
elif pair_sfq == 'sf-q':
cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sfProb_nuvrk'] > 0.5]
cat_neighbors = cat_neighbors[cat_neighbors['sfProb_nuvrk'] < 0.5]
elif pair_sfq == 'q-sf':
cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sfProb_nuvrk'] < 0.5]
cat_neighbors = cat_neighbors[cat_neighbors['sfProb_nuvrk'] > 0.5]
elif pair_sfq == 'q-q':
cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sfProb_nuvrk'] < 0.5]
cat_neighbors = cat_neighbors[cat_neighbors['sfProb_nuvrk'] < 0.5]
coord_neighbors = SkyCoord(np.array(cat_neighbors['RA']) * u.deg, np.array(cat_neighbors['DEC']) * u.deg) # coord of gal2's
coord_gals_z = SkyCoord(np.array(cat_neighbors_z['RA']) * u.deg, np.array(cat_neighbors_z['DEC']) * u.deg) # coord of gal1's
if len(cat_neighbors_z) > 3:
arr = search_around_sky(coord_gals_z, coord_neighbors, max_sep/3600 * u.deg)
sep2d, b = np.unique(np.round(arr[2], 10), return_index=True) # only keep the unique pairs
arr0 = arr[0][b]
arr1 = arr[1][b]
# arr_all = search_around_sky(coord_gals_z, coord_neighbors, aper_size*2/3600 * u.deg)
# a2, b2 = np.unique(np.round(arr_all[2], 10), return_index=True)
# no_pairs_all = len(b2)
gal1_list = arr0[sep2d.value > 2.5 / 3600]
gal2_list = arr1[sep2d.value > 2.5 / 3600]
no_pairs = len(gal1_list)
if no_pairs == 0:
fails += 1
continue
ra1, dec1, f1 = getcoords('20130602A-0003-0016') # b
ra2, dec2, f2 = getcoords('20130602A-0003-0018') # y
frame1 = Frame('20130602A-0003-0016')
frame2 = Frame('20130602A-0003-0018')
f1 /= frame1.exposure
f2 /= frame2.exposure
# get imaging data
# image_data = fetch_imaging_sample()
c = SkyCoord(ra=ra1 * u.degree, dec=dec1 * u.degree)
catalog = SkyCoord(ra=ra2 * u.degree, dec=dec2 * u.degree)
# idx, d2d, d3d = c.match_to_catalog_sky(catalog)
# matches = catalog[idx]
# print len(matches)
# idxc, idxcatalog, d2d, d3d = catalog.search_around_sky(c, 1.5*u.arcsec)
idxc, idxcatalog, d2d, d3d = search_around_sky(c, catalog, 1.5 * u.arcsec)
print(len(idxc))
# plt.hist(d2d*3600.0, bins=30)
# plt.show()
mby = -2.5 * np.log10(f1[idxc] / f2[idxcatalog])
my = -2.5 * np.log10(f2[idxcatalog])
plt.scatter(mby, my)
plt.ylim(plt.ylim()[1], plt.ylim()[0])
plt.grid()
plt.show()
plt.close()
