def get_Q0(lofar_coords, opt_coords, random_coords, radius): """ Compute the Q_0 parameter for a given radius Parameters: ----------- lofar_coords : LOFAR coordinates opt_coords : Optical/NIR coordinates n_random : Number of random sources to generate radius : Radius to search for blanks in the random or random blank catalogue Returns: -------- Q_0 value : Strictly returns Q_0/F(r) from the above equation """ # Match the LOFAR catalogue to the optical/NIR catalogue ind_1l, ind_2o, d2d, d3d = search_around_sky(lofar_coords, opt_coords, radius * u.arcsec) # Number of LOFAR sources without a optical/NIR counterpart within radius nl_no_match = len(lofar_coords) - len(np.unique(ind_1l)) # Match this random catalogue to theoptical/NIR catalogue ind_r, ind_2o, d2d, d3d = search_around_sky(random_coords, opt_coords, radius * u.arcsec) # Number of random sources without an optical/NIR counterpart within radius nr_no_match = len(random_coords) - len(np.unique(ind_r)) q0_value = 1 - (float(nl_no_match) / nr_no_match * (float(len(random_coords)) / len(lofar_coords))) return q0_value, nl_no_match, nr_no_match
def test_regression_4082(): """ Issue: https://github.com/astropy/astropy/issues/4082 """ from astropy.coordinates import search_around_sky, search_around_3d cat = SkyCoord([10.076, 10.00455], [18.54746, 18.54896], unit='deg') search_around_sky(cat[0:1], cat, seplimit=u.arcsec * 60, storekdtree=False) # in the issue, this raises a TypeError # also check 3d for good measure, although it's not really affected by this bug directly cat3d = SkyCoord([10.076, 10.00455]*u.deg, [18.54746, 18.54896]*u.deg, distance=[0.1, 1.5]*u.kpc) search_around_3d(cat3d[0:1], cat3d, 1*u.kpc, storekdtree=False)
def test_regression_4082(): """ Issue: https://github.com/astropy/astropy/issues/4082 """ from astropy.coordinates import search_around_sky, search_around_3d cat = SkyCoord([10.076, 10.00455], [18.54746, 18.54896], unit='deg') search_around_sky(cat[0:1], cat, seplimit=u.arcsec * 60, storekdtree=False) # in the issue, this raises a TypeError # also check 3d for good measure, although it's not really affected by this bug directly cat3d = SkyCoord([10.076, 10.00455]*u.deg, [18.54746, 18.54896]*u.deg, distance=[0.1, 1.5]*u.kpc) search_around_3d(cat3d[0:1], cat3d, 1*u.kpc, storekdtree=False)
def gc_serial(A_coord, B_coord, radius): ''' ''' from astropy.coordinates import search_around_sky def assert_input(A_coord, B_coord, radius): from astropy.coordinates import SkyCoord from astropy.units import Quantity assert isinstance(A_coord,SkyCoord), "Was expecting an ~astropy.coordinates.SkyCoord instance for 'A_coord'." assert isinstance(B_coord,SkyCoord), "Was expecting an ~astropy.coordinates.SkyCoord instance for 'B_coord'." assert isinstance(radius,Quantity), "Was expecting an ~astropy.units.Quantity instance for 'radius'" assert_input(A_coord, B_coord, radius) logging.info("Searching B_coord {1} objects, {0} neighbors.".format(len(B_coord),len(A_coord))) _prau = A_coord.ra.unit _pdecu = A_coord.dec.unit _nrau = B_coord.ra.unit _ndecu = B_coord.dec.unit logging.debug("Unit of coordinates being matched: ({0},{1}) and ({2},{3})".format(_prau,_pdecu,_nrau,_ndecu)) match_A_gc_idx, match_B_gc_idx, match_gc_sep, _d3d = search_around_sky(A_coord, B_coord, radius) from booq.utils import stats _sts = stats.basic(match_gc_sep.value) logging.info("Basic stats of distances between matchings: {}".format(_sts)) assert len(match_A_gc_idx) == len(match_B_gc_idx) return (match_A_gc_idx, match_B_gc_idx, match_gc_sep)
def fill_duplicates(cat, cols): """Since duplicates are excluded in the catalogue (to save time), they must be filled in to create a complete catalogue. Due to nuances in the flagging routine, this must be done iteratively. """ # Fill in `cols` for duplicates by searching for matches in the rest # of the duplicate components. # Need to apply this multiple times because of the duplicate flagging algorithm. missing_comps = cat[(cat.Duplicate_flag >= 1) & np.isnan(cat[cols[0]])] not_missing_comps = cat[(cat.Duplicate_flag >= 1) & ~np.isnan(cat[cols[0]])] missing_coords = SkyCoord(missing_comps["RA"].values, missing_comps["DEC"].values, unit=u.deg) not_missing_coords = SkyCoord(not_missing_comps["RA"].values, not_missing_comps["DEC"].values, unit=u.deg) idx1, idx2, sep, dist = search_around_sky(missing_coords, not_missing_coords, seplimit=2 * u.arcsec) # When multiple matches are found, choose the one with the highest SNR idx1u, idx1c = np.unique(idx1, return_counts=True) idx2u = [ idx2[idx1 == i1][0] if i1c == 1 else idx2[idx1 == i1][final_cat.iloc[idx2[idx1 == i1]]["SNR"].argmax()] for i1, i1c in zip(idx1u, idx1c) ] for col in cols: cat.loc[missing_comps.iloc[idx1].index, col] = (not_missing_comps[col].iloc[idx2].values)
def estimate_q_m(magnitude, bin_list, n_m, coords_small, coords_big, radius=5): """Compute q(m) Estimation of the distribution of real matched sources with respect to a magnitude (normalized to 1). As explained in Fleuren et al. """ assert len(magnitude) == len(coords_big) # Cross match idx_small, idx_big, d2d, d3d = search_around_sky( coords_small, coords_big, radius*u.arcsec) n_xm_small = len((idx_small)) idx = np.unique(idx_big) # Get the distribution of matched sources n_hist_total, _ = np.histogram(magnitude[idx], bin_list) n_hist_total = n_hist_total.astype('float') # Correct probability if there are no sources if len(magnitude[idx]) == 0: n_hist_total = np.ones_like(n_hist_total)*0.5 for n in range(len(n_hist_total)): test = n_hist_total[n] if (test == 0): n_hist_total[n] = 0.5 # Estimate real(m) n_m_nocumsum = np.ediff1d(n_m, to_begin=n_m[0]) real_m = n_hist_total - n_xm_small*n_m_nocumsum*np.pi*(radius)**2 # Remove small negative numbers real_m[real_m <= 0.] = 0. real_m_cumsum = np.cumsum(real_m) return real_m_cumsum/real_m_cumsum[-1]
def estimate_q_m_kde(magnitude, bin_centre, n_m, coords_small, coords_big, radius=5, bandwidth=0.2): """Compute q(m) Estimation of the distribution of real matched sources with respect to a magnitude (normalized to 1). As explained in Fleuren et al. in a non-cumulative fashion using a KDE. For this function we need the centre of the bins instead of the edges. """ assert len(magnitude) == len(coords_big) # Cross match idx_small, idx_big, d2d, d3d = search_around_sky( coords_small, coords_big, radius*u.arcsec) n_small = len(idx_small) idx = np.unique(idx_big) # Get the distribution of matched sources kde_skl_q_m = KernelDensity(bandwidth=bandwidth) kde_skl_q_m.fit(magnitude[idx][:, np.newaxis]) pdf_q_m = np.exp(kde_skl_q_m.score_samples(bin_centre[:, np.newaxis])) n_hist_total = pdf_q_m*len(magnitude[idx])/np.sum(pdf_q_m) # Correct probability if there are no sources ## CHECK if len(magnitude[idx]) == 0: n_hist_total = np.ones_like(n_hist_total)*0.5 # Estimate real(m) real_m = n_hist_total - n_small*n_m*np.pi*radius**2 # Remove small negative numbers ## CHECK real_m[real_m <= 0.] = 0. return real_m/np.sum(real_m)
def plot_one_night(exps, tiledata, night, startdate, center_l=180): import ephem from astropy import units as u from astropy.coordinates import SkyCoord, search_around_sky from matplotlib import pyplot as p startmjd = int( desisurvey.utils.local_noon_on_date( desisurvey.utils.get_date(startdate)).mjd) nightnum = night - startmjd mstarted = (tiledata['PLANNED'] <= nightnum) & (tiledata['PLANNED'] >= 0) tiles = desisurvey.tiles.get_tiles() p.clf() p.subplots_adjust(hspace=0) p.subplots_adjust(left=0.1, right=0.9) programs = ['DARK', 'BRIGHT'] expindex = tiles.index(exps['TILEID']) expnight = exps['MJD'].astype('i4') m = expnight == night medianmjd = np.median(exps['MJD'][m]) mayall = ephem.Observer() config = desisurvey.config.Configuration() coord = SkyCoord(ra=tiles.tileRA * u.deg, dec=tiles.tileDEC * u.deg) mayall.lon = config.location.longitude().to(u.radian).value mayall.lat = config.location.latitude().to(u.radian).value mayall.date = medianmjd + (2400000.5 - 2415020) moon = ephem.Moon() moon.compute(mayall) tile_diameter = config.tile_radius() * 2 for i, prog in enumerate(programs): mprog = prog == tiles.tileprogram mprogstarted = mstarted & mprog p.subplot(len(programs), 1, i + 1) ra = ((tiles.tileRA - (center_l - 180)) % 360) + (center_l - 180) p.plot(ra[mprog], tiles.tileDEC[mprog], '.', color='gray', markersize=1) p.plot(ra[mprogstarted], tiles.tileDEC[mprogstarted], '.', color='green', markersize=5) m = (expnight == night) & (tiles.tileprogram[expindex] == prog) p.plot(ra[expindex[m]], tiles.tileDEC[expindex[m]], 'r-+') idx1, idx2, sep2d, dist3d = search_around_sky(coord[expindex[m]], coord[expindex[m]], tile_diameter * 10) mdiff = expindex[m][idx1] != expindex[m][idx2] if np.sum(mdiff) > 0: print(f'min separation {prog}: {np.min(sep2d[mdiff])}') p.gca().set_aspect('equal') p.plot(((np.degrees(moon.ra) - (center_l - 180)) % 360) + (center_l - 180), np.degrees(moon.dec), 'o', color='yellow', markersize=10, markeredgecolor='black')
def __call__(self, radius=None): """Compute the Q_0 for a given radius (in arcsecs)""" if radius is None: radius = self.radius # Generate random catalogue with n sources as the small one random_small = self.field.random_catalogue(self.n_small) idx_random_small, idx_big, d2d, d3d = search_around_sky( random_small, self.coords_big, radius*u.arcsec) nomatch_random = self.n_small - len(np.unique(idx_random_small)) print('number of random sources with no matches: {:4d}'.format(nomatch_random)) # Compute match in radius idx_small, idx_big, d2d, d3d = search_around_sky( self.coords_small, self.coords_big, radius*u.arcsec) nomatch_small = self.n_small - len(np.unique(idx_small)) print('number of real sources with no matches: {:4d}'.format(nomatch_small)) return (1. - float(nomatch_small)/float(nomatch_random))
def join_table_by_coordinates(table, table_to_join, columns_to_join=None, columns_to_rename=None, max_distance=1.0 / 3600.0, missing_value=np.nan, table_ra_name='RA', table_dec_name='DEC', table_to_join_ra_name='RA', table_to_join_dec_name='DEC', unit='deg'): """ join two table by matching the sky coordinates Examples -------- wise_cols = ('W1_MAG', 'W1_MAG_ERR', 'W2_MAG', 'W2_MAG_ERR') cols_rename = {'W1_MAG':'W1', 'W1_MAG_ERR':'W1ERR', 'W2_MAG':'W2', 'W2_MAG_ERR':'W2ERR'} join_table_by_coordinates(base, wise, wise_cols, cols_rename) """ t1 = table t2 = table_to_join ra1 = table_ra_name dec1 = table_dec_name ra2 = table_to_join_ra_name dec2 = table_to_join_dec_name idx1, idx2 = search_around_sky(SkyCoord(t1[ra1], t1[dec1], unit=unit), SkyCoord(t1[ra1], t1[dec1], unit=unit), Quantity(max_distance, unit=unit))[:2] n_matched = len(idx1) if n_matched: if columns_to_join is None: columns_to_join = t2.colnames if columns_to_rename is None: columns_to_rename = dict() if isinstance(missing_value, dict): missing_value_dict = missing_value missing_value = np.nan else: missing_value_dict = dict() for c2 in columns_to_join: c1 = columns_to_rename.get(c2, c2) if c1 not in t1: t1[c1] = missing_value_dict.get(c1, missing_value) t1[c1][idx1] = t2[c2][idx2] return n_matched
def main( sample, outfile, img_size, img_path, threads=None, parallel=True, all_comps=None, seplimit=90, **kwargs, ): kwargs.update( dict( ir=True, img_size=img_size, radio_path=img_path, ir_path=img_path, all_comps=all_comps, )) if all_comps is not None: seplimit = u.Quantity(seplimit, u.arcsec) coords = SkyCoord(np.array(sample["RA"]), np.array(sample["DEC"]), unit=u.deg) all_coords = SkyCoord(np.array(all_comps["RA"]), np.array(all_comps["DEC"]), unit=u.deg) idx1, idx2, sep, dist = search_around_sky(coords, all_coords, seplimit=seplimit) kwargs.update(comp_idx_map=(idx1, idx2)) with pu.ImageWriter(outfile, 0, img_size, clobber=True) as pk_img: if not parallel: for idx in tqdm(sample.index): out = vlass_preprocessing(idx, sample, **kwargs) if out is not None: pk_img.add(out[1], attributes=out[0]) else: if threads is None: threads = cpu_count() pool = Pool(processes=threads) results = [ pool.apply_async(vlass_preprocessing, args=(idx, sample), kwds=kwargs) for idx in sample.index ] for res in tqdm(results): out = res.get() if out is not None: pk_img.add(out[1], attributes=out[0])
def getPlates(plateFile,cat): plChunk=np.loadtxt(plateFile,skiprows=41,usecols=(3,4)) catPlates=coord.SkyCoord(plChunk[:,0]*u.degree,plChunk[:,1]*u.degree) catObj=coord.SkyCoord(cat[:,0]*u.degree,cat[:,1]*u.degree) objIndex,plateIndex,sep,dist=coord.search_around_sky(catObj,catPlates,1.49*u.degree) # print('objIndex:',objIndex,'plateIndex:',plateIndex) # print('lencat',len(catObj),'objIndex',len(objIndex),'platejIndex',len(plateIndex)) objId,counts=np.unique(objIndex,return_counts=True) # print(len(catObj),len(catPlates)) # print(len(objId),objId,counts) val,countsC=np.unique(counts,return_counts=True) print("objects belonging to",val,"plates, fibers:",countsC) return counts
def overlay_points(neuron: tuple, ed: pu.heatmap, trans: pu.transform, srcs: Table, cata: Table, *args, **kwargs): """Produce a list of (x,y) of offsets sources that are near objects for each source matching up to a neuron. Arguments: neuron {tuple} -- BMU position matching to ed {pu.heatmap} -- Mapped similarity trans {pu.transform} -- Transform matrix for the similarity srcs {Table} -- Catalogue of sources matching those in ed/trans cata {Table} -- Catalogue of sources to overlay """ head = som.file_head target_idx = neuron[1] * head[1] + neuron[0] print("\tSearching for matches") pos_min = np.argmin(ed.data.reshape(ed.data.shape[0], -1), axis=1) mask = np.argwhere(pos_min == target_idx) pix_pos = [] print(f"\tFound {mask.shape}") print("\tIterating and looking for object") for i in mask: row = srcs[i] trans_info = trans.get_neuron_transform(index=i, pos=neuron) # Added to test the individual flipping cases to see what a # problem was. Still not covinced that it is completely correct. # Got to do with a transpose from splitting/concating PINK binaries. # if trans_info[0] == 0: spos = SkyCoord(ra=row['RA'] * u.deg, dec=row['DEC'] * u.deg) res = search_around_sky(spos, cata, seplimit=5 * u.arcmin) for idx in res[1]: offsets = pu.great_circle_offsets(spos, cata[idx], pix_scale=1.5 * u.arcsec) offsets = apply_transform(trans_info, (offsets, )) pix_pos.append(offsets) pix_pos = np.array(pix_pos).reshape( -1, 2) # No idea why the reshape is needed. return pix_pos, mask
def refmatch(self, sourceframe): from astropy.coordinates import SkyCoord, search_around_sky # @UnresolvedImport from astropy import units as u ra1 = sourceframe.stars['alphawin_j2000'] dec1 = sourceframe.stars['deltawin_j2000'] coords = SkyCoord(ra=ra1 * u.degree, dec=dec1 * u.degree) fluxes = sourceframe.stars['flux_auto'] / sourceframe.expt bidx, refidx, _, _ = search_around_sky(coords, self.refcoord, 0.6 * u.arcsec) # create an empty arry and set with nans compfluxes = np.ones(self.nstars) * np.nan compfluxes[refidx] = fluxes[bidx] return compfluxes
def __init__( self, base_catalogue: SkyCoord, match_catalogues: Tuple[SkyCoord, ...], annotation: Annotator, som_set: SOMSet, cpu_cores: int = None, seplimit: Angle = 1 * u.arcminute, progress: bool = False, **ct_kwargs, ): """Create a set of Filters that describe the projection of sources onto their neurons. Other keyword-arguments are passed to `CoordinateTransformer` Arguments: base_catalogue {SkyCoord} -- The central positions of each image that corresponds to the Mapping and Transform of the currect `som_set` match_catalogues {Tuple[SkyCoord,...]} -- Catalogues of positions to project the neurons through. Each catalogue is assumed to correspond to a single channel annotation {Annotator} -- Previously defined annotated SOM som_set {pu.SOMSet} -- Reference to the SOM, Mapping and Transform binary files for the projection Keyword Arguments: cpu_cores {int} -- The number of CPU cores to use while projecting the filters. The default is to use one (and avoid the ProcessPoolExecutor) (default: {None}) seplimit {Angle} -- Matching area for the `search_around_sky` matcher (defaul: {1*astropy.units.arcminute}) progress {bool} -- Enable the `tqdm` progress bar updates (default: {False}) """ self.base_catalogue = base_catalogue self.annotation = annotation self.som_set = som_set self.cpu_cores = cpu_cores self.seplimit = seplimit self.progress = progress self.ct_kwargs = ct_kwargs assert isinstance( match_catalogues, tuple ), f"Expect tuple of SkyCoord catalogues, even if only of length 1. Received object of type {type(match_catalogues)}" self.match_catalogues = match_catalogues self.sky_matches = [ search_around_sky(self.base_catalogue, mc, self.seplimit) for mc in self.match_catalogues ] logger.debug( f"`search_around_sky` completed, number of matches {[len(sk[0]) for sk in self.sky_matches]}" ) self.filters = self.project()
def get_q_m(lofar_coords, opt_coords, rmax, magnitude, bin_list, area, n_m): """ Get q(m)/Q_0 - computes the real(m)/np.cumsum(real(m)) Parameters: ----------- lofar_coords : LOFAR coordinates (SkyCoord object) opt_coords : Optical/NIR coordinates (SkyCoord object) rmax : rmax within which to search for optical sources of a LOFAR source magnitude : magnitude distribution bin_list : List of bin edges - THE SAME BINS USED TO COMPUTE n(m) area : Survey area of the optical/NIR band n_m : output of get_n_m --- n(m) distribution Returns: -------- q(m): Strictly, it returns q(m)/Q_0 """ # Search for all opt/NIR sources within rmax of LOFAR sources ind_l, ind_o, s2d, d3d = search_around_sky(lofar_coords, opt_coords, rmax * u.arcsec) # Get the unique indices into opt_coords that match ind_o_un = np.unique(ind_o) ind_l_un = np.unique(ind_l) print("Unique opt_ind: ", len(ind_o_un)) print("Unique LOFAR ind: ", len(ind_l_un)) # Get the magnitude of the optical/NIR sources that have a match - defined as total(m) total_m, _ = np.histogram(magnitude[ind_o_un], bin_list) # Compute real(m) real_m = np.cumsum(total_m) - len(lofar_coords) * n_m * np.pi * rmax**2 # Remove any possible small negative values? real_m[real_m <= 0.] = 0. # Cumulative sum of the real_m # real_m_cumsum = np.cumsum(real_m) # Return the q(m)/Q_0 value - where Q_0 has yet to be determined (using an iterative approach) return real_m / real_m[-1], np.cumsum(total_m), real_m
def apply_circles(self, lon=None, lat=None): # read the list of circles from the file try: hdul = fits.open(self.filename) except: raise Exception(f"Failed to find/open circles file: {self.filename}") data = hdul[1].data try: circle_lon = data[self.col_lon] circle_lat = data[self.col_lat] except: raise Exception(f"Failed to find circle columns: {self.filename}[1][{self.col_lon},{self.col_lat}]") if isinstance(self.radius, float): circle_radius = np.full(len(data), self.radius) elif isinstance(self.radius, str): try: circle_radius = data[self.radius] except: raise Exception(f"Failed to find circle column: {self.filename}[1][{self.radius}]") else: raise Exception(f"Cannot interpret radius: {self.radius}") hdul.close() ### need to use a single radius for all objects seplimit = u.Quantity(circle_radius.max(), unit='deg') coords_c = SkyCoord(circle_lon, circle_lat, frame="icrs", unit="deg") coords_t = SkyCoord(lon, lat, frame="icrs", unit="deg") idx_t, idx_c, d2d, d3d = search_around_sky(coords_t, coords_c, seplimit) ## filter the list of matches to use the per-circle radius i_t_ok = [ i_t for i_t,i_c,d in zip(idx_t, idx_c, d2d) if d.deg < circle_radius[i_c]] m_mask = np.zeros(len(lon), np.bool) m_mask[i_t_ok] = True return m_mask
def indices_xfind_coords( coords1, coords2, maxdist=1 * u.arcsec, obstime=None, ): """Indices of X-Find coords. Returns ------- idx1 : integer array idx2 : integer array indices into `other` for all coordinates which have a "match" in `catalog`. info : dict Useful information # TODO See Also -------- :func:`~astropy.coordinates.search_around_sky` :func:`~astropy.coordinates.search_around_3d` """ if u.get_physical_type(maxdist.unit) == "angle": idx1, idx2, sep2d, dist3d = coord.search_around_sky( coords1, coords2, seplimit=maxdist, ) elif u.get_physical_type(maxdist.unit) == "length": idx1, idx2, sep2d, dist3d = coord.search_around_3d( coords1, coords2, distlimit=maxdist, ) info = {"sep2d": sep2d, "dist3d": dist3d} return idx1, idx2, info
def coord_matching(first_coords, second_coords, match_sep): """ Performs a coordinates cross-match of the positions between two SkyCoord objects coord_matching(tomatch_coords, catalog_coords, match_sep) Input: first_coords: First set of coordinates - a SkyCoord object second_coords: Second set of coordinates - a SkyCoord object match_sep: Separation to search within - in arcseconds Output: indx1: Indices into first_coords that have matched with coords2 - Array indx2: Indices into second_coords that have matched with coords1 - Array sep2d: On-sky separation between the coordinates - (Angle object) """ # Perform the catalog match indx1, indx2, sep2d, dist3d = search_around_sky(first_coords, second_coords, seplimit=match_sep * u.arcsec) return indx1, indx2, sep2d
def best_host(components, ir_srcs, sep_limit): # A host is most likely when it aligns with a radio components likely_host = False sep_limit = u.Quantity(sep_limit, u.arcsec) sky_matches = search_around_sky(components, ir_srcs, seplimit=15 * u.arcsecond) if len(sky_matches[0]) > 0: best_match = np.argmin(sky_matches[2]) best_ir_match = ir_srcs[sky_matches[1][best_match]] # Obtained from cross-matching a shifted catalogue to WISE if sky_matches[2][best_match] < 3.4 * u.arcsecond: # ax.scatter( # best_ir_match.ra, # best_ir_match.dec, # transform=ax.get_transform("world"), # color="white", # s=22, # ) likely_host = True return likely_host
# Reading SDSS Stripe 82 clean data S82 = pd.read_csv('../../data/clean_stripe82.csv') # Reading SDSS DR7 clean data DR7 = pd.read_csv('../../data/clean_dr7.csv') # Rename the ra, dec coordinates from the 2 different data sets DR7 = DR7.rename(index=str, columns={'ra': 'ra_DR7', 'dec': 'dec_DR7'}) S82 = S82.rename(index=str, columns={'ra': 'ra_S82', 'dec': 'dec_S82'}) # Match data attributes in the 2 data sets using astropy's SkyCoord COORD1 = SkyCoord(DR7['ra_DR7'], DR7['dec_DR7'], frame='icrs', unit='deg') COORD2 = SkyCoord(S82['ra_S82'], S82['dec_S82'], frame='icrs', unit='deg') IDX1, IDX2, OTHER1, OTHER2 = search_around_sky(COORD1, COORD2, seplimit=0.5 * u.arcsec) # Generating columns for the matched X_TRAIN = [] for i in range(len(IDX1)): result = DR7.iloc[IDX1[i]].append(S82.iloc[IDX2[i]]) X_TRAIN.append(result) X_TRAIN = pd.concat(X_TRAIN, axis=1) X_TRAIN = X_TRAIN.T # Remove the unnecessary part X_TRAIN = X_TRAIN.loc[:, ~X_TRAIN.columns.str.contains('^Unnamed')] X_TRAIN = X_TRAIN.drop(['BH_mass'], axis=1) X_TRAIN = X_TRAIN.astype({'ID': int})
def xmatch_cat(table1=None, table2=None, radec1=None, radec2=None, nthneighbor=None, multimatch=False, seplimit=10.0, selfmatch=False, colnames_radec1=['ra', 'dec'], colnames_radec2=['ra', 'dec'], units_radec1=['degree', 'degree'], units_radec2=['degree', 'degree'], stats=False, debug=False, verbose=False, method=False): """RA, Dec nearest xmatch for two lists; returns pointers nearest match input can be an astropy table or zipped radec as a list e.g. c = zip([1],[1]) radec1 = zip(ra1 , dec1) radec1 = np.column_stack(ra1, dec1)) Self match notes: """ import numpy as np import matplotlib.pyplot as plt from astropy.table import Table, hstack from astropy.coordinates import SkyCoord from astropy.coordinates import search_around_sky, match_coordinates_sky from astropy import units as u from astropy.stats import mad_std, median_absolute_deviation if verbose or debug: print('__file__:', __file__) print('__name__:', __name__) try: if 'filename' in table1.meta: print('table1.filename:', table1.meta['filename']) except: print("table1 has no metadata or table1.meta['filename']") if verbose or debug: print('colnames_radec1:', colnames_radec1) table1.info() # selfmatch does not need a 2nd table if not selfmatch: try: if 'filename' in table2.meta: print('table2.filename:', table2.meta['filename']) except: print("table2 has no metadata or table2.meta['filename']") if verbose or debug: print('colnames_radec2:', colnames_radec2) table2.info() if selfmatch: table2 = table1 colnames_radec2 = colnames_radec1 if nthneighbor is None: nthneighbor = 2 if nthneighbor is None: nthneighbor = 1 ra1 = table1[colnames_radec1[0]] dec1 = table1[colnames_radec1[1]] if verbose or debug: print('table1: ', colnames_radec1[0], table1[colnames_radec1[0]].unit) print('table1: ', colnames_radec1[1], table1[colnames_radec1[1]].unit) ra2 = table2[colnames_radec2[0]] dec2 = table2[colnames_radec2[1]] if verbose or debug: print('table2: ', colnames_radec2[0], table2[colnames_radec2[0]].unit) print('table2: ', colnames_radec2[1], table2[colnames_radec2[1]].unit) if stats or verbose or debug: print('RA1 range:', np.min(ra1), np.max(ra1)) print('Dec1 range:', np.min(dec1), np.max(dec1)) print('RA1 range:', np.min(ra2), np.max(ra2)) print('Dec1 range:', np.min(dec2), np.max(dec2)) skycoord1 = SkyCoord(ra1, dec1, unit=units_radec1, frame='icrs') skycoord2 = SkyCoord(ra2, dec2, unit=units_radec2, frame='icrs') # idx is an integer array into the second cordinate array to get the # matched points for the second coordindate array. # Shape of idx matches the first coordinate array idx1 = [] idx2 = [] if not method: if not multimatch: idx2, d2d, d3d = \ match_coordinates_sky(skycoord1, skycoord2, nthneighbor=nthneighbor) if multimatch: idx1, idx2, d2d, d3d = \ search_around_sky(skycoord1, skycoord2, seplimit * u.arcsec) # alternative 'method' form if method: if not multimatch: idx2, d2d, d3d = \ skycoord1.match_to_catalog_sky(skycoord2, nthneighbor=nthneighbor) if multimatch: idx1, idx2, d2d, d3d = \ skycoord1.search_around_sky(skycoord2, seplimit * u.arcsec) # compute the separations and if not multimatch: separation = skycoord1.separation(skycoord2[idx2]) dra, ddec = \ skycoord1.spherical_offsets_to(skycoord2[idx2]) if multimatch: separation = skycoord1[idx1].separation(skycoord2[idx2]) dra, ddec = \ skycoord1[idx1].spherical_offsets_to(skycoord2[idx2]) if stats or verbose or debug: print('multimatch:', multimatch) print('seplimit:', seplimit) print('len(table1):', len(table1)) print('len(table2):', len(table2)) print('len(idx1):', len(idx1)) print('len(idx2):', len(idx2)) print('idxmatch range:', np.min(idx2), np.max(idx2)) print('d2d range:', np.min(d2d), np.max(d2d)) print('d2d range:', np.min(d2d).arcsec, np.max(d2d).arcsec) print('d2d median:', np.median(d2d).arcsec) median_separation = np.median(separation).arcsec mad_std_separation = mad_std(separation.arcsec) print('dR range (arcsec):', np.min(separation.arcsec), np.max(separation.arcsec)) print('dR mean, std (arcsec):', np.mean(separation).arcsec, np.std(separation).arcsec) print('dR median, mad_std (arcsec):', median_separation, mad_std_separation) print() median_dra = np.median(dra).arcsec mad_std_dra = mad_std(dra.arcsec) print('dRA min, max:', np.min(dra).arcsec, np.max(dra).arcsec) print('dRA mean, std:', np.mean(dra).arcsec, np.std(dra).arcsec) print('dRA median, mad_std:', median_dra, mad_std_dra) print() median_ddec = np.median(ddec).arcsec mad_std_ddec = mad_std(ddec.arcsec) print('dDec min, max:', np.min(ddec).arcsec, np.max(ddec).arcsec) print('dDec mean, std:', np.mean(ddec).arcsec, np.std(ddec).arcsec) print('dDec median, mad_std:', median_ddec, mad_std_ddec) print() # convert to arcsec for convenience separation = separation.arcsec dr = d2d.arcsec dra = dra.arcsec ddec = ddec.arcsec # return dra, ddec, dr in arcsec # as a list or could be dict; check if scales from 10^3 -> 10^6 -> 10^9 drplus = [dra, ddec, dr] if debug or verbose: print(len(idx2), len(dr)) print(len(drplus), len(drplus[0]), len(drplus[1]), len(drplus[2])) # could add option to return dr, dra, ddec if not multimatch: return idx2, dr, dra, ddec if multimatch: return (idx1, idx2), dr, dra, ddec
def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # pylint: disable=no-member # Try to read cosmology from catalog, otherwise defualts to WMAP7 try: cosmo = catalog_instance.cosmology except AttributeError: cosmo = WMAP7 # Create interpolation tables for efficient computation of sigma crit z = np.linspace(0, self.zmax, int(self.zmax)*100) d1 = cosmo.angular_diameter_distance(z) # in Mpc angular_diameter_distance = interp1d(z, d1, kind='quadratic') d2 = cosmo.comoving_transverse_distance(z) # in Mpc comoving_transverse_distance = interp1d(z, d2, kind='quadratic') # Now figure out the lenses, for the validation data available, # each have slightly non-trivial cuts, so we do them separately... not totally ideal if self.data == 'sdss_lowz': # Singh et al (2015) (http://adsabs.harvard.edu/abs/2015MNRAS.450.2195S) measurements on the SDSS LOWZ sample. res = catalog_instance.get_quantities(['redshift_true', 'ra', 'dec', 'shear_1', 'shear_2', 'mag_true_i_sdss', 'mag_true_z_sdss','mag_true_g_sdss', 'mag_true_r_sdss']) # Compute mask for lowz sample # These cuts are defined in section 3 of https://arxiv.org/pdf/1509.06529.pdf # and summarised here: http://www.sdss.org/dr14/algorithms/boss_galaxy_ts/#TheBOSSLOWZGalaxySample # Definition of auxiliary colors: cperp = (res['mag_true_r_sdss'] - res['mag_true_i_sdss']) - (res['mag_true_g_sdss'] - res['mag_true_r_sdss'])/4.0 - 0.18 cpar = 0.7*(res['mag_true_g_sdss'] - res['mag_true_r_sdss']) + 1.2*((res['mag_true_r_sdss'] - res['mag_true_i_sdss'])-0.18) # LOWZ selection cuts: mask_lens = np.abs(cperp) < 0.2 # color boundaries mask_lens &= res['mag_true_r_sdss'] < (13.5 + cpar/0.3) # sliding magnitude cut mask_lens &= (res['mag_true_r_sdss'] > 16) &(res['mag_true_r_sdss'] < 19.6) # Additional redshift cuts used in Singh et al. (2015) mask_lens &= (res['redshift_true'] > self.zmin_l) & (res['redshift_true'] < self.zmax_l) Mask_lens = [mask_lens] fig = plt.figure() if self.data == 'cfhtlens': res = catalog_instance.get_quantities(['redshift_true', 'ra', 'dec', 'shear_1', 'shear_2', 'Mag_true_g_lsst_z0', 'Mag_true_r_lsst_z0']) Mr_min = np.array([-21.0,-22.0,-23.0,-24.0]) Mr_max = np.array([-20.0,-21.5,-22.5,-23.5]) blue_frac = np.array([0.7,0.32,0.11,0.03])*100 gr = res['Mag_true_g_lsst_z0'] - res['Mag_true_r_lsst_z0'] # larger number means redder Mask_lens = [] for i in range(4): mask_lens = (res['redshift_true']>self.zmin_l) & (res['redshift_true']<self.zmax_l) & (res['Mag_true_r_lsst_z0']>Mr_min[i]) & (res['Mag_true_r_lsst_z0']<Mr_max[i]) gr_threshold = np.percentile(gr[mask_lens], blue_frac[i]) Mask_lens.append(mask_lens & (gr>gr_threshold)) Mask_lens.append(mask_lens & (gr<gr_threshold)) fig1 = plt.figure(1, figsize=(12,9)) fig2 = plt.figure(2, figsize=(12,5)) if self.data == 'sdss_main': res = catalog_instance.get_quantities(['redshift_true', 'ra', 'dec', 'shear_1', 'shear_2', 'mag_true_i_sdss', 'mag_true_z_sdss','mag_true_g_sdss', 'mag_true_r_sdss', 'stellar_mass_bulge', 'stellar_mass_disk','Mag_true_g_sdss_z0','Mag_true_r_sdss_z0']) gr = res['Mag_true_g_sdss_z0'] - res['Mag_true_r_sdss_z0'] # larger number means redder sm = res['stellar_mass_bulge'] + res['stellar_mass_disk'] SM_min = np.array([10,10.7,11.2,11.6]) SM_max = np.array([10.4,11.0,11.4,15.0]) Mask_lens = [] for i in range(4): mask_lens = (res['redshift_true']>self.zmin_l) & (res['redshift_true']<self.zmax_l) & (res['mag_true_r_sdss']< 17.7) & (np.log10(sm)>SM_min[i]) & (np.log10(sm)<SM_max[i]) Mask_lens.append(mask_lens & (gr>0.7)) # for the data, 0.7 is used for k-correct colors at z=0.1 Mask_lens.append(mask_lens & (gr<0.7)) fig1 = plt.figure(1, figsize=(12,9)) fig2 = plt.figure(2, figsize=(12,5)) # Computing mask for source sample, this only serves to keep the number of galaxies managable mask_source = (res['redshift_true'] > self.zmin_s) & (res['redshift_true'] < self.zmax_s) inds = np.where(mask_source)[0] if len(inds) > int(self.max_background_galaxies): mask_source[inds[np.random.choice(len(inds), size=len(inds) - int(self.max_background_galaxies), replace=False)]] = False coords = SkyCoord(ra=res['ra']*u.degree, dec=res['dec']*u.degree) coords_s = coords[mask_source] # run gammat in thin redshift bins, loop over lens bins of different stellar mass and colors for i in range(len(Mask_lens)): nlens = len(np.where(Mask_lens[i])[0]) / catalog_instance.sky_area with open(os.path.join(output_dir, 'galaxy_density_'+str(self.data)+'.dat'), 'a') as f: f.write('{} \n'.format(nlens)) # Create astropy coordinate objects coords_l = coords[Mask_lens[i]] # Search for neighbours idx1, idx2, sep2d, _ = search_around_sky(coords_l, coords_s, 3.*u.deg) # Computing sigma crit for each pair zl = res['redshift_true'][Mask_lens[i]][idx1] zs = res['redshift_true'][mask_source][idx2] # Warning: this assumes a flat universe # See http://docs.astropy.org/en/v0.3/_modules/astropy/cosmology/core.html#FLRW.angular_diameter_distance_z1z2 dm1 = comoving_transverse_distance(zl) dm2 = comoving_transverse_distance(zs) angular_diameter_distance_z1z2 = u.Quantity((dm2 - dm1)/(1. + zs), u.Mpc) sigcrit = cst.c**2 / (4.*np.pi*cst.G) * angular_diameter_distance(zs) / \ ((1. + zl)**2. * angular_diameter_distance_z1z2 * angular_diameter_distance(zl)) # NOTE: the validation data is in comoving coordinates, the next few # lines take care of proper unit conversions # Apply unit conversion to obtain sigma crit in h Msol /pc^2 (comoving) cms = u.Msun / u.pc**2 sigcrit = sigcrit*(u.kg/(u.Mpc* u.m)).to(cms) / cosmo.h # Computing the projected separation for each pairs, in Mpc/h (comoving) r = sep2d.rad*angular_diameter_distance(zl)*(1. + zl) * cosmo.h # Computing the tangential shear thetac = np.arctan2((coords_s[idx2].dec.rad - coords_l[idx1].dec.rad) / np.cos((coords_s[idx2].dec.rad + coords_l[idx1].dec.rad) / 2.0),coords_s[idx2].ra.rad - coords_l[idx1].ra.rad) gammat = -(res['shear_1'][mask_source][idx2] * np.cos(2*thetac) - res['shear_2'][mask_source][idx2] * np.sin(2*thetac)) # Binning the tangential shear bins = np.logspace(np.log10(self.Rmin), np.log10(self.Rmax), self.nR, endpoint=True) counts = np.histogram(r, bins=bins)[0] gt, b = np.histogram(r, bins=bins, weights=gammat*sigcrit) rp = 0.5*(b[1:]+b[:-1]) gt = gt/counts outfile = os.path.join(output_dir, 'DS_'+str(self.data)+'_'+str(i)+'.dat') np.savetxt(outfile, np.vstack((rp, gt)).T) if self.data == 'sdss_lowz': ax = plt.subplot(111) plt.errorbar(self.validation_data[:,0], self.validation_data[:,1], yerr=self.validation_data[:,2], label='SDSS LOWZ from Singh et al. (2015)',c='k', lw=1, marker='.', fmt='.', capthick=0.8, capsize=2.2) plt.loglog(rp, gt, label=catalog_name) plt.title('Lens number density: '+str(nlens)[:4]+' per sq. deg') ax.set_xlabel('$r_p$ [Mpc/h]') ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$') ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3) ax.set_ylim(0.5, 100) if self.data == 'cfhtlens': ii = np.mod(i,2) iii = int(i/2) plt.figure(1) ax = plt.subplot(2,2,iii+1) if ii==0: plt.loglog(rp, gt, label=str(Mr_min[int(i/2)])+'< Mr < '+str(Mr_max[int(i/2)])+'; red; '+catalog_name, lw=2, color='r', alpha=0.5) plt.errorbar(self.validation_data[:,0]/1000*(7./10.), self.validation_data[:,iii*2+1]/(7./10.), color='darkred', lw=2, marker='x', fmt='.', label='Velander et al. (2013)') plt.text(self.Rmin*0.7*1.5, 1.5,'Red: '+str(nlens)[:4]+' per sq. deg') else: plt.loglog(rp, gt, label=str(Mr_min[int(i/2)])+'< Mr < '+str(Mr_max[int(i/2)])+'; blue', lw=2, color='b', alpha=0.5) plt.errorbar(self.validation_data[:,0]/1000*(7./10.), self.validation_data[:,iii*2+2]/(7./10.), color='darkblue', lw=2, marker='x', fmt='.') plt.title('Lens number density: '+str(nlens)[:4]+' per sq. deg') plt.text(self.Rmin*0.7*1.5, 1.0,'Blue: '+str(nlens)[:4]+' per sq. deg') ax.legend() ax.set_xlabel('$r_p$ [Mpc/h]') ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$') ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3) ax.set_ylim(0.5, 1000) plt.tight_layout() plt.figure(2) ax = plt.subplot(1,2,ii+1) plt.loglog(rp, gt, label='['+str(Mr_min[int(i/2)])+', '+str(Mr_max[int(i/2)])+']') if ii==0: plt.title('red') else: plt.title('blue') if i==(len(Mask_lens)-1): plt.legend() ax.set_xlabel('$r_p$ [Mpc/h]') ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$') ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3) ax.set_ylim(0.5, 500) if self.data=='sdss_main': ii = np.mod(i,2) iii = int(i/2) plt.figure(1) ax = plt.subplot(2,2,iii+1) if ii==0: plt.loglog(rp, gt, label=str(SM_min[int(i/2)])+'< log10(M*) < '+str(SM_max[int(i/2)])+'; red; '+catalog_name, lw=2, color='r', alpha=0.5) plt.errorbar(self.validation_data[:15,0], self.validation_data[ii*15:(ii+1)*15,int(i/2)*4+1], yerr=self.validation_data[ii*15:(ii+1)*15,int(i/2)*4+2], color='darkred', lw=2, marker='x', fmt='.', label='Mandelbaum et al. (2016)') plt.text(self.Rmin*0.7*1.5, 1.5,'Red: '+str(nlens)[:4]+' per sq. deg') else: plt.loglog(rp, gt, label=str(SM_min[int(i/2)])+'< log10(M*) < '+str(SM_max[int(i/2)])+'; blue', lw=2, color='b', alpha=0.5) plt.errorbar(self.validation_data[:15,0], self.validation_data[ii*15:(ii+1)*15,int(i/2)*4+1], yerr=self.validation_data[ii*15:(ii+1)*15,int(i/2)*4+2], color='darkblue', lw=2, marker='x', fmt='.') plt.text(self.Rmin*0.7*1.5, 1,'Blue: '+str(nlens)[:4]+' per sq. deg') ax.legend() ax.set_xlabel('$r_p$ [Mpc/h]') ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$') ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3) ax.set_ylim(0.5, 1000) plt.tight_layout() plt.figure(2) ax = plt.subplot(1,2,ii+1) plt.loglog(rp, gt, label='['+str(SM_min[int(i/2)])+', '+str(SM_max[int(i/2)])+']') if ii==0: plt.title('red') else: plt.title('blue') if i==(len(Mask_lens)-1): plt.legend() ax.set_xlabel('$r_p$ [Mpc/h]') ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$') ax.set_xlim(self.Rmin*0.7, self.Rmax*1.3) ax.set_ylim(0.5, 500) plt.tight_layout() print(self.data) if self.data=='cfhtlens' or self.data=='sdss_main': fig1.savefig(os.path.join(output_dir, 'delta_sigma_'+str(catalog_name)+'1.png')) plt.close(fig1) fig2.savefig(os.path.join(output_dir, 'delta_sigma_'+str(catalog_name)+'2.png')) plt.close(fig2) else: fig.savefig(os.path.join(output_dir, 'delta_sigma_'+str(catalog_name)+'.png')) plt.close(fig) return TestResult(inspect_only=True)
cat_neighbors = cat_neighbors[cat_neighbors['sSFR_med'] > -11] elif pair_sfq == 'q-q': cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sSFR_med'] < -11] cat_neighbors = cat_neighbors[cat_neighbors['sSFR_med'] < -11] coord_neighbors = SkyCoord( np.array(cat_neighbors['ra']) * u.deg, np.array(cat_neighbors['dec']) * u.deg) coord_gals_z = SkyCoord( np.array(cat_neighbors_z['ra']) * u.deg, np.array(cat_neighbors_z['dec']) * u.deg) if len(cat_neighbors_z) > 5: len_cat_nei.append(len(cat_neighbors_z)) arr = search_around_sky(coord_gals_z, coord_neighbors, 10. / 3600 * u.deg) arr_all = search_around_sky(coord_gals_z, coord_neighbors, aper_size * 2 / 3600 * u.deg) gal1_list = arr[0][arr[2].value > 2.5 / 3600] gal2_list = arr[1][arr[2].value > 2.5 / 3600] no_pairs = len(gal1_list) no_pairs_all = len(arr_all[0]) if no_pairs > 0: print('no of objects', len(cat_neighbors_z), 'no. of close pairs', no_pairs, 'no. of pairs ', no_pairs_all) else: continue # calculate deltaz for k in range(len(gal1_list)): if gal1_list[k] < gal2_list[k]: # only unique pairs
def run_on_single_catalog(self, catalog_instance, catalog_name, output_dir): # pylint: disable=no-member # Try to read cosmology from catalog, otherwise defualts to WMAP7 try: cosmo = catalog_instance.cosmology except AttributeError: cosmo = WMAP7 # Create interpolation tables for efficient computation of sigma crit z = np.linspace(0, self.zmax, self.zmax * 100) d1 = cosmo.angular_diameter_distance(z) # in Mpc angular_diameter_distance = interp1d(z, d1, kind='quadratic') d2 = cosmo.comoving_transverse_distance(z) # in Mpc comoving_transverse_distance = interp1d(z, d2, kind='quadratic') res = catalog_instance.get_quantities([ 'redshift_true', 'ra', 'dec', 'shear_1', 'shear_2', 'convergence', 'mag_true_i_sdss', 'mag_true_z_sdss', 'mag_true_g_sdss', 'mag_true_r_sdss' ]) # Compute mask for lowz sample # These cuts are defined in section 3 of https://arxiv.org/pdf/1509.06529.pdf # and summarised here: http://www.sdss.org/dr14/algorithms/boss_galaxy_ts/#TheBOSSLOWZGalaxySample # Definition of auxiliary colors: cperp = (res['mag_true_r_sdss'] - res['mag_true_i_sdss']) - ( res['mag_true_g_sdss'] - res['mag_true_r_sdss']) / 4.0 - 0.18 cpar = 0.7 * ( res['mag_true_g_sdss'] - res['mag_true_r_sdss']) + 1.2 * ( (res['mag_true_r_sdss'] - res['mag_true_i_sdss']) - 0.18) # LOWZ selection cuts: mask_lowz = np.abs(cperp) < 0.2 # color boundaries mask_lowz &= res['mag_true_r_sdss'] < (13.5 + cpar / 0.3 ) # sliding magnitude cut mask_lowz &= (res['mag_true_r_sdss'] > 16) & (res['mag_true_r_sdss'] < 19.6) # Counting the number density of LOWZ galaxies nlens = len(np.where(mask_lowz)[0]) / catalog_instance.sky_area with open(os.path.join(output_dir, 'galaxy_density.dat'), 'a') as f: f.write('{} \n'.format(nlens)) # Additional redshift cuts used in Singh et al. (2015) mask_lowz &= (res['redshift_true'] > 0.16) & (res['redshift_true'] < 0.36) # Computing mask for source sample, this only serves to keep the number # of neighbours manageable mask_source = res['redshift_true'] > self.zcut_background inds = np.where(mask_source)[0] if len(inds) > int(self.max_background_galaxies): mask_source[inds[np.random.choice( len(inds), size=len(inds) - int(self.max_background_galaxies), replace=False)]] = False # Create astropy coordinate objects coords = SkyCoord(ra=res['ra'] * u.degree, dec=res['dec'] * u.degree) coords_l = coords[mask_lowz] coords_s = coords[mask_source] # Search for neighbours idx1, idx2, sep2d, _ = search_around_sky(coords_l, coords_s, 2. * u.deg) # Computing sigma crit for each pair zl = res['redshift_true'][mask_lowz][idx1] zs = res['redshift_true'][mask_source][idx2] # Warning: this assumes a flat universe # See http://docs.astropy.org/en/v0.3/_modules/astropy/cosmology/core.html#FLRW.angular_diameter_distance_z1z2 dm1 = comoving_transverse_distance(zl) dm2 = comoving_transverse_distance(zs) angular_diameter_distance_z1z2 = u.Quantity((dm2 - dm1) / (1. + zs), u.Mpc) sigcrit = cst.c**2 / (4.*np.pi*cst.G) * angular_diameter_distance(zs) / \ ((1. + zl)**2. * angular_diameter_distance_z1z2 * angular_diameter_distance(zl)) # NOTE: the validation data is in comoving coordinates, the next few # lines take care of proper unit conversions # Apply unit conversion to obtain sigma crit in h Msol /pc^2 (comoving) cms = u.Msun / u.pc**2 sigcrit = sigcrit * (u.kg / (u.Mpc * u.m)).to(cms) / cosmo.h # Computing the projected separation for each pairs, in Mpc/h (comoving) r = sep2d.rad * angular_diameter_distance(zl) * (1. + zl) * cosmo.h # Computing the tangential shear thetac = np.arctan2( (coords_s[idx2].dec.rad - coords_l[idx1].dec.rad) / np.cos( (coords_s[idx2].dec.rad + coords_l[idx1].dec.rad) / 2.0), coords_s[idx2].ra.rad - coords_l[idx1].ra.rad) gammat = -(res['shear_1'][mask_source][idx2] * np.cos(2 * thetac) - res['shear_2'][mask_source][idx2] * np.sin(2 * thetac)) # Binning the tangential shear bins = np.logspace(np.log10(0.05), 1, 17, endpoint=True) counts = np.histogram(r, bins=bins)[0] gt, b = np.histogram(r, bins=bins, weights=gammat * sigcrit) rp = 0.5 * (b[1:] + b[:-1]) # Outputs the number of background galaxies in each bins and checks that # that number is sufficient. with open(os.path.join(output_dir, 'galaxy_count.dat'), 'a') as f: f.write('{} {} {}\n'.format(rp, counts, gt)) if counts.min() < self.min_count_per_bin: return TestResult( score=np.nan, passed=False, summary="Not enough background sources to compute delta sigma") gt = gt / counts fig = plt.figure() ax = plt.subplot(111) plt.loglog(rp, gt, label='LOWZ-like sample from ' + catalog_name) plt.errorbar(self.validation_data[:, 0], self.validation_data[:, 1], yerr=self.validation_data[:, 2], label='SDSS LOWZ from Singh et al. (2015)') plt.title( 'Number density {}/deg$^2$ vs 57/deg$^2$ for LOWZ'.format(nlens)) ax.set_xlabel('$r_p$ [Mpc/h]') ax.set_ylabel(r'$\Delta \Sigma [h \ M_\odot / pc^2]$') ax.legend() ax.set_xlim(0.05, 10) ax.set_ylim(0.5, 100) fig.savefig( os.path.join(output_dir, 'delta_sigma_{}.png'.format(catalog_name))) plt.close(fig) return TestResult(inspect_only=True)
def filter_cmb( path="/home/chema/Escritorio/datos/LFI_SkyMap_030-BPassCorrected_1024_R3.00_full.fits", f=30, fwhm=32.65, cut=4, print_sources=False): star_time = time.time() # Loading of the FITS file with the image of the CMB and change the units to Jy. Specify the path to the file, # the frequency and the fwhm of the instrument. f, fwhm = f * u.GHz, fwhm * u.arcmin # Input data # The first element of the array is the one with the T map. object_map = Fitsmap.from_file(path, freq=f, fwhm=fwhm)[0] print("--- Data loaded ---") # Changing data from K to Jy object_map.to_Jy() # --- Parameters that characterize the patches that will be used to cover the CMB map on # which we will use the filter --- patch_size = 256 # --- Computing the center of the patches to cover all the sphere --- nside = 8 npix = hp.nside2npix(nside) centers = [hp.pix2vec(nside, i) for i in range(npix)] # List with the position vectors of the npix of side = 8 that will be the center of the patches. lon, lat = hp.vec2ang(np.array(centers), lonlat=True) # Center vector to lon and lat in deg. npatches = len(centers) # --- Filtering every patch and getting all the candidates of compact sources, gathering all in a table --- peaks = [] for i in trange(npatches): coord = SkyCoord(l=lon[i], b=lat[i], frame='galactic', unit=u.deg) patch = object_map.patch(coord, npix=patch_size) filtered_patch = patch.matched(fwhm=object_map.fwhm) std = filtered_patch.std() threshold = cut * std indices = peak_local_max(filtered_patch.datos, min_distance=2, threshold_abs=threshold, exclude_border=True, indices=True) for x, y in indices: coordinate = filtered_patch.pixel_coordinate(x, y) # Estimation of the flux error s = (object_map.fwhm / filtered_patch.pixsize).si.value w = np.arange(0, patch_size, 1, float) v = w[:, np.newaxis] r = np.sqrt((w - x)**2 + (v - y)**2) m = (r >= 3 * s) & (r <= 5 * s) peak = { 'Longitude': coordinate.l.value, 'Latitude': coordinate.b.value, 'Flux': filtered_patch.datos[x, y], 'Flux_error': filtered_patch.datos[m].std(), 'SNR': filtered_patch.datos[x, y] / std } peaks.append(peak) print("--- Map filtered ---") table = Table(peaks) coordinates = SkyCoord( l=np.array(table['Longitude'].tolist()), b=np.array(table['Latitude'].tolist()), frame='galactic', unit=u.deg ) # Coordinates of each peak in the galactic system (in angles). # --- Searching for repetitions in our catalogue --- resolution_max = object_map.fwhm.to(u.arcmin) ind1, ind2, dist, d3d = search_around_sky(coordinates, coordinates, 3 / 4 * resolution_max) correspondence_table = Table() correspondence_table['Index 1'] = ind1 correspondence_table['Index 2'] = ind2 correspondence_table['Separation'] = dist final_table = Table() # --- Resolution of coincidences --- while len(correspondence_table) > len(final_table): skipped = [] final_table = Table() for i in trange(len(table)): if i == 0: final_table = Table(rows=table[i], names=('Longitude', 'Latitude', 'Flux', 'Flux_error', 'SNR')) if i in skipped: continue x = correspondence_table[correspondence_table['Index 1'] == i] if len(x) == 1: if i == 0: continue else: final_table.add_row([ table['Longitude'][i], table['Latitude'][i], table['Flux'][i], table['Flux_error'][i], table['SNR'][i] ]) else: skipped = np.append(skipped, np.array(x['Index 2'].tolist())) coord = SkyCoord(l=table['Longitude'].tolist()[i], b=table['Latitude'].tolist()[i], frame='galactic', unit=u.deg) patch = object_map.patch(coord, npix=int(patch_size / 2), deltatheta_deg=14.658 / 2) filtered_patch = patch.matched(fwhm=object_map.fwhm) std = filtered_patch.std() threshold = cut * std indices = peak_local_max(filtered_patch.datos, min_distance=2, threshold_abs=threshold, exclude_border=True, indices=True) for x, y in indices: s = (object_map.fwhm / filtered_patch.pixsize).si.value if (x >= patch_size/4 - 3/4*s) & (x <= patch_size/4 + 3/4*s) & \ (y >= patch_size/4 - 3/4*s) & (y <= patch_size/4 + 3/4*s): coordinate = patch.pixel_coordinate(x, y) # Estimation of the flux error w = np.arange(0, int(patch_size / 2), 1, float) v = w[:, np.newaxis] r = np.sqrt((w - x)**2 + (v - y)**2) m = (r >= 3 * s) & (r <= 5 * s) peak = { 'Longitude': coordinate.l.value, 'Latitude': coordinate.b.value, 'Flux': filtered_patch.datos[x, y], 'Flux_error': filtered_patch.datos[m].std(), 'SNR': filtered_patch.datos[x, y] / std } final_table.add_row([ peak['Longitude'], peak['Latitude'], peak['Flux'], peak['Flux_error'], peak['SNR'] ]) if i == 0: final_table.remove_row(0) coordinates = SkyCoord(l=np.array(final_table['Longitude'].tolist()), b=np.array(final_table['Latitude'].tolist()), frame='galactic', unit=u.deg) ind1, ind2, dist, d3d = search_around_sky(coordinates, coordinates, 3 / 4 * resolution_max) correspondence_table = Table() correspondence_table['Index 1'] = ind1 correspondence_table['Index 2'] = ind2 correspondence_table['Separation'] = dist table = final_table print("--- Duplicated sources removed ---") # Saving the table with the catalogue. gal_coords = SkyCoord(l=np.array(table['Longitude'].tolist()), b=np.array(table['Latitude'].tolist()), frame='galactic', unit=u.deg) eq_coords = gal_coords.icrs table.add_column(col=eq_coords.ra, index=2, name='RA') table.add_column(col=eq_coords.dec, index=2, name='DEC') table['Longitude'].unit, table['Latitude'].unit = 'deg', 'deg' table['Flux'].unit, table['Flux_error'].unit = 'Jy', 'Jy' table['RA'].unit, table['DEC'].unit = 'deg', 'deg' table.write('catalogue_' + str(f.value) + 'GHz.fits', overwrite=True) print("--- Data saved ---") # Saving a picture with the localization of all the sources. if print_sources: hp.projscatter(table['Longitude'].data.data, table['Latitude'].data.data, lonlat=True, coord='G', color='r', marker='.') hp.graticule() hp.projscatter(table['Longitude'].data.data, table['Latitude'].data.data, lonlat=True, coord='G', color='r', marker='.') plt.savefig('Detected_Sources_' + str(f) + '_SNR' + str(cut) + '.png') print(str(len(table)) + ' compact sources detected.') print('Execution time: ' + str(time.strftime("%M:%S", time.gmtime(time.time() - star_time))) + ' (min:s)')
def calculate_sim_images(simfolder, vis=None, baseimage=None, baseimage_file=None, repeat=10, basename=None, savefile=None, fov_scale=1.5, second_check=False, detection_threshold=2.5, aperture_size=None, aperture_scale=6.0, plot=False, snr_mode='peak', debug=False, overwrite=True, method='sep', snr=[1,20], seplimit_scale=0.5, seplimit_arcsec=None, mask=None, **kwargs): """simulation the completeness of source finding algorithm mode: peak: snr is the peak value integrated: snr is the integrated value seplimit: in arcsec The second_check is set to false to get the completeness across the whole SNR """ if not isinstance(baseimage, FitsImage): if os.path.isfile(baseimage_file): baseimage = FitsImage(baseimage_file) else: raise ValueError("No valid baseimage or file has been provided!") # recalibrate the std based one the masked image baseimage.imagemask = baseimage.mask_image(mask=baseimage.find_structure(sigma=3.0)) baseimage.imstat() # known_sources = source_finder(baseimage, fov_scale=fov_scale) if basename is None: basename = baseimage.name # define the statistical variables boosting_table = Table(names=['snr_peak_input', 'snr_peak_found', 'flux_input','flux_aperture','flux_gaussian'], dtype=['f8', 'f8', 'f8', 'f8', 'f8']) comp_table = Table(names=['snr_peak', 'is_recovered'], dtype=['f8', 'int']) fake_table = Table(names=['snr_peak', 'is_fake'], dtype=['f8','int']) for run in np.arange(repeat): if debug: print("calculating run: {}".format(run)) # try: if True: simimage_imagefile = "{basename}.run{run}.image.fits".format(basename=basename, run=run) simimage_sourcefile = "{basename}.run{run}.txt".format(basename=basename, run=run) simimage_fullpath = os.path.join(simfolder, simimage_imagefile) simimage_sourcefile_fullpath = os.path.join(simfolder, simimage_sourcefile) # print("simulated image:", simimage_fullpath) # print("simulated sources:", simimage_sourcefile_fullpath) ##### for new simulations # sources_input = Table.read(simimage_sourcefile_fullpath, format='ascii') # sources_input_coords = SkyCoord(ra=sources_input['ra[deg]']*u.deg, # dec=sources_input['dec[deg]']*u.deg) # sources_input_flux = sources_input['flux[Jy]'] #convert to Jy ##### for old simulations sources_input = Table.read(simimage_sourcefile_fullpath, format='ascii') sources_input_coords = SkyCoord(ra=sources_input['ra[arcsec]']*u.arcsec, dec=sources_input['dec[arcsec]']*u.arcsec) sources_input_flux = sources_input['flux[mJy]']/1000 #convert to Jy simimage = FitsImage(simimage_fullpath) sources_found = source_finder(simimage, detection_threshold=detection_threshold, method=method, mask=mask) sources_found_coords = SkyCoord(ra=sources_found['ra']*u.deg, dec=sources_found['dec']*u.deg) if seplimit_scale is not None: seplimit = seplimit_scale * simimage.bmaj*3600 * u.arcsec if seplimit_arcsec is not None: seplimit = seplimit_arcsec * u.arcsec if len(sources_input) > 0: if len(sources_found) > 0: idx_input, idx_found, d2d, d3d = search_around_sky(sources_input_coords, sources_found_coords, seplimit) idx_input_comp = np.array(list(set(range(len(sources_input))) - set(idx_input)), dtype=int) idx_found_comp = np.array(list(set(range(len(sources_found))) - set(idx_found)), dtype=int) # calculate the flux boosting snr_found_boosting = sources_found['peak_flux'][idx_found] / baseimage.std flux_aperture, flux_aperture_err = measure_flux(simimage, detections=sources_found[idx_found], aperture_size=aperture_size, aperture_scale=aperture_scale, minimal_aperture_size=simimage.bmaj*2*3600, method='single-aperture') flux_gaussian, flux_gaussian_err = measure_flux(simimage, detections=sources_found[idx_found], method='gaussian') flux_input = sources_input_flux[idx_input] snr_input_boosting = sources_input_flux[idx_input] / baseimage.std boosting_single = Table([snr_found_boosting, snr_input_boosting, snr_found_boosting, flux_input, flux_aperture, flux_gaussian], names=['snr_peak','snr_peak_input','snr_peak_found','flux_input','flux_aperture','flux_gaussian']) boosting_table = vstack([boosting_table, boosting_single]) # calculate the completeness snr_input_completeness = sources_input_flux/baseimage.std is_recovered = np.zeros_like(snr_input_completeness).astype(int) is_recovered[idx_input] = 1 is_recovered[idx_input_comp] = 0 comp_single = Table([snr_input_completeness, is_recovered], names=['snr_peak', 'is_recovered']) comp_table = vstack([comp_table, comp_single]) # calculate the false detection snr_found_fakeness = sources_found['peak_flux']/baseimage.std is_fake = np.zeros_like(snr_found_fakeness).astype(int) is_fake[idx_found] = 0 is_fake[idx_found_comp] = 1 fake_single = Table([snr_found_fakeness, is_fake], names=['snr_peak', 'is_fake']) fake_table = vstack([fake_table, fake_single]) # except: # print('Failed in run {}'.format(run)) # continue if savefile: boosting_table.write(savefile+'_boosting.dat', format='ascii', overwrite=overwrite) comp_table.write(savefile+'_completeness.dat', format='ascii', overwrite=overwrite) fake_table.write(savefile+'_fake.dat', format='ascii', overwrite=overwrite) return boosting_table
def add_to_hdf(hdf, Z_MIN = 0.1, Z_MAX = 7.1, MATCH_TOL = 2.0*u.arcsec): """Generate Myers + SDSS_BOSS QSO catalog from Myers and DR12 files This routine reads in the SDSS/BOSS specObj and PhotPosPlate files and pulls out all QSOs. This is then matched amd merged with the Myers QSO compilation with duplicates removed. It adds this catalog to the 'quasars' field of the hdf5 object hdf Requires that that environment varialble RAW_IGMSPEC be set to the top directory where the SDSS/BOSS files and Myers files live. Parameters ---------- hdf : hdf5 object database to hold QSO catalog Z_MIN : float, optional [default Z_MIN = 0.1] minimum QSO redshift applied to the catalog Z_MAX : float. optimal [default Z_MAX = 7.1] maximum QSO redshift applied to the catalo MATCH_TOL : quantity, optional [default 2.0*u.arcsec] matching radius between Myers and SDSS/BOSS catalogs Returns ------- None : None Examples -------- >>> add_to_hdf(hdf) None Notes ----- In the 'quasars' table added to the hdf5 object hdf, tags with the SDSS_BOSS_ prefix are the SDSS/BOSS tags, and tags with MYERS_ are from the Myers catalog. We also add the following tags 'SDSS_BOSS_MYERS_FLAG' = set to either SDSS_BOSS_MYERS, SDSS_BOSS_ONLY, or MYERS_ONLY 'RA', 'DEC' = our best knowledge of the coordinates (PLUG_RA, PLUG_DEC for SDSS/BOSS, otherwise Myers) 'SOURCEBIT' = Myers catalog sourcebits, with a new 2**19 sourcebit for objects in SDSS_BOSS_ONLY objects 'ZEM' = Our best knwoledge of the redshift ( either MYERS_ZEM or SDSS/BOSS redshift for SDSS_BOSS_ONLY objects 'ZEM_SOURCE' = The source of the redshift following the Myers classification with an additional SDSS_BOSS_ONLY """ import json from astropy.table import Column, hstack, vstack from astropy.coordinates import SkyCoord, search_around_sky ## SDSS/BOSS data stuff specfile = os.getenv('RAW_IGMSPEC') + '/SDSS_BOSS/specObj-dr12_trim.fits' spec = Table.read(specfile) # Read in select columns from DR12 photometry. This and the file above are aligned posfile = os.getenv('RAW_IGMSPEC') + '/SDSS_BOSS/photoPosPlate-dr12_trim.fits' phot = Table.read(posfile) # Trim to QSO, Specprimary, spec2d called it a QSO, redshift flag cuts, sanity check on coords itrim = (spec['SPECPRIMARY'] == 1) & \ [('QSO' in q) for q in spec['CLASS']] & \ (spec['ZWARNING'] < 5) & \ (spec['PLUG_RA'] >= 0.0) & (spec['PLUG_RA'] <= 360.0) & \ (np.abs(spec['PLUG_DEC']) <= 90.0) spec = spec[itrim] phot = phot[itrim] sdss_boss1 = hstack([spec, phot], join_type='exact') # Add SDSS prefix to all SDSS tags for key in sdss_boss1.keys(): sdss_boss1.rename_column(key, 'SDSS_BOSS_' + key) # Read in the Myers file, match it to Myers sweeps photometry # Myers master QSO catalog ADM_file = os.getenv('RAW_IGMSPEC') + '/Myers/GTR-ADM-QSO-master-wvcv.fits.gz' ADM_qso = Table.read(ADM_file) head1 = fits.open(ADM_file)[1].header DATE = head1['DATE'] # Photometry for Myers QSO catalog. This file is not aligned with the catalog file, i.e. it is a # superset that includes the catalog file. For that reason we need to match and tack on photometry ADM_sweep_file = os.getenv('RAW_IGMSPEC') + '/Myers/GTR-ADM-QSO-master-sweeps-Feb5-2016.fits' ADM_sweep = Table.read(ADM_sweep_file) c_qso = SkyCoord(ra=ADM_qso['RA'], dec=ADM_qso['DEC'],unit ='deg') c_swp = SkyCoord(ra=ADM_sweep['RA'], dec=ADM_sweep['DEC'], unit='deg') ## Create an aligned Table for matching photometry from sweeps nqso = len(ADM_qso) qso_phot = Table(np.repeat(np.zeros_like(ADM_sweep[0]), nqso)) # Rename the RA and DEC qso_phot.rename_column('RA', 'RA_sweep') qso_phot.rename_column('DEC', 'DEC_sweep') # Cull out the keys which already exist in the ADM_qso Table (except # for the RA and DEC, which we renamed) dupe_keys = list(set(ADM_qso.keys()) & set(qso_phot.keys())) qso_phot.remove_columns(dupe_keys) # Match the Myers catalog to the Myers sweeps idx, d2d, d3d = c_qso.match_to_catalog_sky(c_swp) # Currently using 1.0" for matching, as for the SDSS objects, these will mostly be the exact # same coordinates. itrim = (d2d <= 1.0 * u.arcsec) qso_phot[:][itrim] = ADM_sweep[:][idx[itrim]] ADM_qso = hstack([ADM_qso, qso_phot], join_type='exact') # Trim to only spectroscopic objects ispec = spectro_myers(ADM_qso) ADM_qso = ADM_qso[ispec] # assign best redshifts to ZEM tag zbest_myers(ADM_qso) # Add MYERS prefix to all MYERS tags for key in ADM_qso.keys(): ADM_qso.rename_column(key, 'MYERS_' + key) # Now we meatch the SDSS/BOSS and Myers catalogs to create one master QSO catalog # # There are three groups of objects, 1) SDSS-MYERS match, 2) SDSS only, 3) Myers only. # Deal with each in turn. # 1) SDSS-MYERS match. Add Myers tags to the SDSS structure c_sdss = SkyCoord(ra=sdss_boss1['SDSS_BOSS_PLUG_RA'], dec=sdss_boss1['SDSS_BOSS_PLUG_DEC'], unit='deg') c_myers = SkyCoord(ra=ADM_qso['MYERS_RA'], dec=ADM_qso['MYERS_DEC'], unit='deg') isdss, imyers, d2d, _ = search_around_sky(c_sdss, c_myers, MATCH_TOL) sdss_myers = hstack([sdss_boss1[isdss], ADM_qso[imyers]], join_type='exact') sdss_myers['SDSS_BOSS_MYERS_FLAG'] = 'SDSS_BOSS_MYERS' sdss_myers['RA'] = sdss_myers['SDSS_BOSS_PLUG_RA'] # SDSS/BOSS Plug coords most accurate sdss_myers['DEC'] = sdss_myers['SDSS_BOSS_PLUG_DEC'] sdss_myers['SOURCEBIT'] = sdss_myers['MYERS_SOURCEBIT'] sdss_myers['ZEM'] = sdss_myers['MYERS_ZEM'] sdss_myers['ZEM_SOURCE'] = sdss_myers['MYERS_ZEM_SOURCE'] # 2) SDSS only # Find the SDSS objects that have no match in the Myers catalog inomatch = np.ones(len(c_sdss), dtype=bool) inomatch[isdss] = False sdss_only = sdss_boss1[inomatch] sdss_only['SDSS_BOSS_MYERS_FLAG'] = 'SDSS_BOSS_ONLY' sdss_only['RA'] = sdss_only['SDSS_BOSS_PLUG_RA'] sdss_only['DEC'] = sdss_only['SDSS_BOSS_PLUG_DEC'] sdss_only['SOURCEBIT'] = 2 ** 19 # New source bit for SDSS only objects sdss_only['ZEM'] = sdss_only['SDSS_BOSS_Z'] sdss_only['ZEM_SOURCE'] = 'SDSS_BOSS_ONLY' # 3) Myers only # Find the Myers objects that have no match in SDSS/BOSS inomatch = np.ones(len(c_myers), dtype=bool) inomatch[imyers] = False myers_only = ADM_qso[inomatch] myers_only['SDSS_BOSS_MYERS_FLAG'] = 'MYERS_ONLY' myers_only['RA'] = myers_only['MYERS_RA'] myers_only['DEC'] = myers_only['MYERS_DEC'] myers_only['SOURCEBIT'] = myers_only['MYERS_SOURCEBIT'] myers_only['ZEM'] = myers_only['MYERS_ZEM'] myers_only['ZEM_SOURCE'] = myers_only['MYERS_ZEM_SOURCE'] sdss_myers_out = vstack([sdss_myers, sdss_only, myers_only]) # Cut down ztrim = (sdss_myers_out['ZEM'] >= Z_MIN) & (sdss_myers_out['ZEM'] <= Z_MAX) coordtrim = (sdss_myers_out['RA'] >= 0.0) & (sdss_myers_out['RA'] <= 360.0) & (np.abs( sdss_myers_out['DEC']) <= 90.0) keep = ztrim & coordtrim sdss_myers_out = sdss_myers_out[keep] # Clean out unicode sbu.clean_table_for_hdf(sdss_myers_out) hdf['quasars'] = sdss_myers_out hdf['quasars'].attrs['MYERS_DATE'] = DATE # Myers dict mdict = myers_dict() hdf['quasars'].attrs['MYERS_DICT'] = json.dumps(ltu.jsonify(mdict)) return None
def check_splus(ra, dec, tilesep_deg, catsep_arcsec): print() obj = create_sky_object(ra, dec) print("Input location (RA,DEC): ", obj.ra.value[0], 'deg', obj.dec.value[0], 'deg') # read SPLUS pointings file splus_tiles_file = 'data/all_pointings.csv' tiles = Table.read(splus_tiles_file, format="csv") tiles_coords = SkyCoord(ra=tiles['RA'], dec=tiles['DEC'], unit=(u.hourangle, u.degree)) # read IDR2 pointings file idr2_tiles_file = 'data/idr2_pointings.csv' idr2_tiles = Table.read(idr2_tiles_file, format="csv") sky_radius_deg = tilesep_deg * u.degree print("Searching all S-PLUS pointings...") idx_data1, idx_data2, d2d, d3d = search_around_sky(obj, tiles_coords, sky_radius_deg) if len(d2d) > 0: for i in range(0, len(d2d)): location_in_idr2 = '(NOT IN DR2)' sel = (idr2_tiles['FIELD'] == tiles['NAME'][idx_data2[i]]) nfound = np.sum(sel) if nfound > 0: location_in_idr2 = '(IDR2)' print( "Your location is within{0:4.1f} degrees of the center of pointings:" .format(tilesep_deg)) print("{0:s} {1:s}/{2:s} (separation:{3:5.2f})".format( location_in_idr2, tiles['PID'][idx_data2[i]], tiles['NAME'][idx_data2[i]], d2d[i].value)) else: print("Nothing found.") print() print("Searching all iDR2 objects...") infile = 'Catalogs/merged_id.fits' poscat = Table.read(infile, format='fits') obscoords = SkyCoord(ra=poscat['RA'], dec=poscat['Dec'], unit=(u.degree, u.degree)) sky_radius_deg = (catsep_arcsec / 3600.) * u.degree idx_data1, idx_data2, d2d, d3d = search_around_sky(obj, obscoords, sky_radius_deg) sor = np.argsort(d2d) idx_data1 = idx_data1[sor] idx_data2 = idx_data2[sor] d2d = d2d[sor] d3d = d3d[sor] if len(d2d) > 0: for i in range(0, len(d2d)): print( "Object {0:s} in area/tile {1:7s}/{2:12s} with separation: {3:5.2f} arcsec)" .format(str(poscat['ID'][idx_data2[i]]), poscat['SURVEY'][idx_data2[i]], poscat['FIELD'][idx_data2[i]], d2d[i].value * 3600.)) else: print("Nothing found.") return
def add_spectra(base, specs): if 'coord' in specs.colnames: del specs['coord'] specs.sort(['ZQUALITY', 'SPEC_Z']) specs.reverse() specs = add_skycoord(specs) specs_idx, base_idx, sep, _ = search_around_sky( specs['coord'], base['coord'], 20.0 * astropy.units.arcsec) # pylint: disable=E1101 sep = sep.arcsec if (np.ediff1d(specs_idx) < 0).any(): sorter = specs_idx.argsort() specs_idx = specs_idx[sorter] base_idx = base_idx[sorter] sep = sep[sorter] del sorter base['index'] = np.arange(len(base)) base['no_spec_yet'] = True specs['matched_idx'] = -1 specs_idx_edges = np.flatnonzero( np.hstack(([1], np.ediff1d(specs_idx), [1]))) for i, j in zip(specs_idx_edges[:-1], specs_idx_edges[1:]): spec_idx_this = specs_idx[i] possible_match = base['REMOVE', 'is_galaxy', 'r_mag', 'radius', 'no_spec_yet', 'index'][base_idx[i:j]] possible_match['sep'] = sep[i:j] possible_match[ 'sep_norm'] = possible_match['sep'] / possible_match['radius'] if Query('sep < 1', ~Query('no_spec_yet')).count(possible_match) > 0: specs['matched_idx'][spec_idx_this] = -2 # duplicated specs possible_match = possible_match[possible_match['no_spec_yet']] if not len(possible_match): continue larger_search_r = build._get_spec_search_radius( specs['SPEC_Z'][spec_idx_this]) for q, sorter in ( (Query('REMOVE == 0', ~Query('is_galaxy'), 'sep < 0.5'), 'sep'), (Query('REMOVE == 0', 'is_galaxy', 'sep_norm < 1', 'sep < {:g}'.format(larger_search_r)), 'r_mag'), (Query('REMOVE == 0', 'is_galaxy', 'sep < 3'), 'r_mag'), (Query('REMOVE == 0', ~Query('is_galaxy'), 'sep < 3'), 'sep'), (Query('REMOVE > 0', ~Query('is_galaxy'), 'sep < 0.5'), 'sep'), (Query('REMOVE > 0', 'is_galaxy', 'sep_norm < 1', 'sep < {:g}'.format(larger_search_r)), 'r_mag'), (Query('REMOVE > 0', 'is_galaxy', 'sep < 3'), 'r_mag'), (Query('REMOVE > 0', ~Query('is_galaxy'), 'sep < 3'), 'sep'), ): mask = q.mask(possible_match) if mask.any(): possible_match_this = possible_match[mask] matched_base_idx = possible_match_this['index'][ possible_match_this[sorter].argmin()] specs['matched_idx'][spec_idx_this] = matched_base_idx base['no_spec_yet'][matched_base_idx] = False break del base['index'], base['no_spec_yet'], specs[ 'coord'], specs_idx, base_idx, sep, specs_idx_edges specs.sort('matched_idx') start_idx = np.flatnonzero(specs['matched_idx'] > -1)[0] for col in tuple(SPECS_COLUMNS) + ('SPEC_REPEAT', 'OBJ_NSAID'): col_base = (col + '_spec') if col in ('RA', 'DEC') else col base[col_base][specs['matched_idx'] [start_idx:]] = specs[col][start_idx:] specs_warn = specs[np.flatnonzero(specs['matched_idx'] > -2)[0]:start_idx] specs_warn = specs_warn[specs_warn['SPEC_REPEAT'] != 'GAMA'] if len(specs_warn) > 100: logging.warning( 'More than 100 spec objects have no match... something is very wrong!' ) else: for spec in specs_warn: logging.warning( 'No photo obj matched to {} spec obj {} ({}, {})'.format( spec['TELNAME'], spec['SPECOBJID'], spec['RA'], spec['DEC'])) del specs['matched_idx'] return base
def run_bayes(frb_tbl_file, galaxy_cat_file, prior, outfile, frb_ee, galaxy_coords=None, max_radius=20., mag_lim=None, ncpu=15, multi=True): """ Run the Bayes analysis Args: frb_tbl_file: galaxy_cat_file: prior: outfile: frb_ee: galaxy_coords: max_radius: Returns: """ # Defs sb_defs = cosmos_defs() # Load FRBs print("Loading...") sb_frbs = pandas.read_csv(frb_tbl_file, index_col=0) nFRB = len(sb_frbs) # frb_ee if isinstance(frb_ee, dict): frb_ee_list = [frb_ee] * nFRB elif isinstance(frb_ee, str): frb_ee_list = [] if frb_ee == 'loc_sig': for kk in range(nFRB): frb_ee_list.append({ 'a': sb_frbs.iloc[kk].loc_sig, 'b': sb_frbs.iloc[kk].loc_sig, 'theta': 0. }) else: raise IOError("Bad frb_ee") # Load Galaxies sb_galaxies = pandas.read_feather(galaxy_cat_file) # Cut sb_galaxies = sb_galaxies[np.isfinite(sb_galaxies.a_image) & np.isfinite(sb_galaxies.mag_best)] # Extra bits sb_galaxies['half_light'] = sb_galaxies.a_image * sb_defs['plate_scale'] sb_galaxies[sb_defs['filter']] = sb_galaxies.mag_best # Cut on mag? if mag_lim is not None: keep = sb_galaxies.mag_best < mag_lim sb_galaxies = sb_galaxies[keep] # Coordinates (this is slow) print("Building coords...") frb_coords = SkyCoord(ra=sb_frbs.frb_ra, dec=sb_frbs.frb_dec, unit='deg') if galaxy_coords is None: galaxy_coords = SkyCoord(ra=sb_galaxies.ra, dec=sb_galaxies.dec, unit='deg') # Find the candidates print("Finding candidates...") idx1, idx2, sep2d, _ = search_around_sky(galaxy_coords, frb_coords, 15 * units.arcsec) # Slice the tables print("Slicing...") list_candidates = [] #nFRB = 2000 for kk in range(nFRB): if (kk % 1000) == 0: print('kk: ', kk) in_idx2 = np.where(idx2 == kk)[0] gd_gal = idx1[in_idx2] close_galaxies = sb_galaxies.iloc[gd_gal][ [ #'mag_best', #'a_image', #'kron_radius', #'b_image', #'ra', 'dec', 'class_star', 'half_light', sb_defs['filter'], 'ra', 'dec' ]] # Extras close_galaxies['separation'] = sep2d[gd_gal].to('arcsec').value close_galaxies['coords'] = galaxy_coords[gd_gal] # list_candidates.append(close_galaxies) # Loop me! print('Starting to loop!') # Build if multi: pool = multiprocessing.Pool(processes=ncpu) results = [ pool.apply_async( main_calc, args=(idx_FRB, frb_coords[idx_FRB], frb_ee_list[idx_FRB], list_candidates[idx_FRB], sb_defs, prior, max_radius)) for idx_FRB in range(nFRB) ] output = [p.get() for p in results] idx = [item[1] for item in output] all_tbls = np.array([item[0] for item in output], dtype=object) all_tbls = all_tbls[idx] # Expunge the None's gd_tbl = np.array( [False if item is None else True for item in all_tbls]) gd_idx = np.arange(all_tbls.size)[gd_tbl] all_tbls = all_tbls[gd_tbl] for kk in range(all_tbls.size): all_tbls[kk]['iFRB'] = gd_idx[kk] else: embed(header='broken') all_tbls = [] for idx_FRB in range(nFRB): sv_tbl = main_calc(frb_coords[idx_FRB], sb_galaxies, galaxy_coords, FRB, frbA, sb_defs, prior) sv_tbl['iFRB'] = idx_FRB all_tbls.append(sv_tbl) # Finish final_tbl = pandas.concat(all_tbls) final_tbl.to_csv(outfile) print("Wrote: {}".format(outfile))
cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sfProb_nuvrk'] > 0.5] cat_neighbors = cat_neighbors[cat_neighbors['sfProb_nuvrk'] > 0.5] elif pair_sfq == 'sf-q': cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sfProb_nuvrk'] > 0.5] cat_neighbors = cat_neighbors[cat_neighbors['sfProb_nuvrk'] < 0.5] elif pair_sfq == 'q-sf': cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sfProb_nuvrk'] < 0.5] cat_neighbors = cat_neighbors[cat_neighbors['sfProb_nuvrk'] > 0.5] elif pair_sfq == 'q-q': cat_neighbors_z = cat_neighbors_z[cat_neighbors_z['sfProb_nuvrk'] < 0.5] cat_neighbors = cat_neighbors[cat_neighbors['sfProb_nuvrk'] < 0.5] coord_neighbors = SkyCoord(np.array(cat_neighbors['RA']) * u.deg, np.array(cat_neighbors['DEC']) * u.deg) # coord of gal2's coord_gals_z = SkyCoord(np.array(cat_neighbors_z['RA']) * u.deg, np.array(cat_neighbors_z['DEC']) * u.deg) # coord of gal1's if len(cat_neighbors_z) > 3: arr = search_around_sky(coord_gals_z, coord_neighbors, max_sep/3600 * u.deg) sep2d, b = np.unique(np.round(arr[2], 10), return_index=True) # only keep the unique pairs arr0 = arr[0][b] arr1 = arr[1][b] # arr_all = search_around_sky(coord_gals_z, coord_neighbors, aper_size*2/3600 * u.deg) # a2, b2 = np.unique(np.round(arr_all[2], 10), return_index=True) # no_pairs_all = len(b2) gal1_list = arr0[sep2d.value > 2.5 / 3600] gal2_list = arr1[sep2d.value > 2.5 / 3600] no_pairs = len(gal1_list) if no_pairs == 0: fails += 1 continue
ra1, dec1, f1 = getcoords('20130602A-0003-0016') # b ra2, dec2, f2 = getcoords('20130602A-0003-0018') # y frame1 = Frame('20130602A-0003-0016') frame2 = Frame('20130602A-0003-0018') f1 /= frame1.exposure f2 /= frame2.exposure # get imaging data # image_data = fetch_imaging_sample() c = SkyCoord(ra=ra1 * u.degree, dec=dec1 * u.degree) catalog = SkyCoord(ra=ra2 * u.degree, dec=dec2 * u.degree) # idx, d2d, d3d = c.match_to_catalog_sky(catalog) # matches = catalog[idx] # print len(matches) # idxc, idxcatalog, d2d, d3d = catalog.search_around_sky(c, 1.5*u.arcsec) idxc, idxcatalog, d2d, d3d = search_around_sky(c, catalog, 1.5 * u.arcsec) print(len(idxc)) # plt.hist(d2d*3600.0, bins=30) # plt.show() mby = -2.5 * np.log10(f1[idxc] / f2[idxcatalog]) my = -2.5 * np.log10(f2[idxcatalog]) plt.scatter(mby, my) plt.ylim(plt.ylim()[1], plt.ylim()[0]) plt.grid() plt.show() plt.close()