def mask_catalog_columns(catalog, idx, columns=None, catname=None, new_columns=None, catalog_kwargs=None): """ Mask all of the rows in a table (or subset of columns in a table) with a masked index and (optionally) rename the columns. Parameters ---------- catalog: `~astropy.table.Table` or `~Catalog` Catalog or Table to be masked idx: `~numpy.ma.array` Masked array of indices to use for updating the catalog columns: list of strings, optional Columns to include in the masked table. If ``columns is None`` (default) then all of the columns are included in the masked catalog. catname: str, optional Name of catalog. This is only necessary if you with to rename the columns of the catalog for stacking later. See ``new_columns`` for more. new_columns: list of strings, optional New names for the columns. This may be useful if you are combining catalogs and want to standardize the column names. If ``new_columns is None`` (default) then if a ``catname`` is provided all of the columns are renamed to 'columnname_catname', otherwise the original column names are used. catalog_kwargs: dict If the result is desired to be a `~astropyp.catalog.Catalog` then these are the kwargs to specify when initializing the catalog (for example names of the ra,dec,x,y columns). Otherwise an `~astropy.table.Table` is returned Returns ------- tbl: `~astropy.table.Table` or `~astropyp.catalog.Catalog` Catalog created by applying the masked index. The type of object returned depends on if ``catalog_kwargs is None`` (default), which returns a Table. Otherwise a Catalog is returned. """ from astropy.table import Table if isinstance(catalog, Catalog): tbl = catalog.sources else: tbl = catalog new_tbl = Table(masked=True) if columns is None: columns = tbl.columns.keys() if new_columns is None: if catname is None: new_columns = columns else: new_columns = ['{0}_{1}'.format(col,catname) for col in columns] for n, col in enumerate(columns): new_tbl[new_columns[n]] = misc.update_ma_idx(tbl[col], idx) if catalog_kwargs is not None: new_tbl = Catalog(new_tbl, **catalog_kwargs) return new_tbl
def merge_catalogs(self, good_indices=None, save_catalog=True): """ Combine catalogs from each image into a master catalog that matches all of the source (by default, although specifying ``good_indices`` allows to only merge a subset of each catalog) Parameters ---------- good_indices: string or list of arrays, optional If good_indices is a string, index ``ccd.indices[good_indices]`` will be used for each CCD to filter the sources. Otherwise good_indices should be a list of indices to use for each CCD. If no god_indices are specified then all sources will be merged save_catalog: bool Whether or not to save the catalog Result ------ catalog: `~astropyp.catalog.Catalog` Catalog of merged positions and indices for the sources in each image. """ import astropyp.catalog from astropyp.utils.misc import update_ma_idx from astropy import table from astropy.extern import six if good_indices is None: all_ra = [ccd.catalog.ra for ccd in self.ccds] all_dec = [ccd.catalog.dec for ccd in self.ccds] else: if isinstance(good_indices, six.string_types): good_indices = [ccd.indices[good_indices] for ccd in self.ccds] all_ra = [ccd.catalog.ra[good_indices[n]] for n, ccd in enumerate(self.ccds)] all_dec = [ccd.catalog.dec[good_indices[n]] for n, ccd in enumerate(self.ccds)] indices, matched = astropyp.catalog.get_merged_indices(all_ra, all_dec) all_x = [] all_y = [] for n,ccd in enumerate(self.ccds): ccd.indices['merge'] = indices[n] if n!=self.ref_index: x0,y0 = self.tx_solutions[ (n,self.ref_index)].transform_coords(ccd.catalog) else: x0,y0 = (ccd.catalog.x, ccd.catalog.y) if good_indices is not None: x0 = x0[good_indices[n]] y0 = y0[good_indices[n]] all_x.append(x0) all_y.append(y0) all_x = [update_ma_idx(all_x[n], self.ccds[n].indices['merge']) for n in self.ccd_indices] all_y = [update_ma_idx(all_y[n], self.ccds[n].indices['merge']) for n in self.ccd_indices] sources = table.Table(masked=True) sources['x'],sources['y'] = astropyp.catalog.combine_coordinates( all_x, all_y, 'mean') for n,ccd in enumerate(self.ccds): x_col = 'x_{0}'.format(n) y_col = 'y_{0}'.format(n) sources[x_col] = all_x[n] sources[y_col] = all_y[n] if good_indices is None: src_idx = self.ccds[n].catalog['src_idx'] else: src_idx = self.ccds[n].catalog['src_idx'][good_indices[n]] src_idx = update_ma_idx(src_idx, self.ccds[n].indices['merge']) sources['idx_{0}'.format(n)] = src_idx x_diff = sources[x_col]-sources['x'] y_diff = sources[y_col]-sources['y'] logger.info('x rms: {0}'.format( np.sqrt(np.sum(x_diff)**2/len(sources)))) logger.info('y rms: {0}'.format( np.sqrt(np.sum(y_diff)**2/len(sources)))) catalog = astropyp.catalog.Catalog(sources) if save_catalog: self.merged_catalog = catalog return catalog
def perform_psf_photometry(self, separation=None, verbose=False, fit_position=True, pos_range=0, indices=None, kd_tree=None, exptime=None, pool_size=None, stack_method='mean', save_catalog=True, single_thread=False): """ Perform PSF photometry on all of the sources in the catalog, or if indices is specified, a subset of sources. Parameters ---------- separation: float, optional Separation (in pixels) for members to be considered part of the same group *Default=1.5*psf width* verbose: bool, optional Whether or not to show info about the fit progress. *Default=False* fit_position: bool, optional Whether or not to fit the position along with the amplitude of each source. *Default=True* pos_range: int, optional Maximum number of pixels (in image pixels) that a sources position can be changed. If ``pos_range=0`` no bounds will be set. *Default=0* indices: `~numpy.ndarray` or string, optional Indices for sources to calculate PSF photometry. It is often advantageous to remove sources with bad pixels and sublinear flux to save processing time. All sources not included in indices will have their psf flux set to NaN. This can either be an array of indices for the positions in self.catalog or the name of a saved index in self.indices """ import astropyp.catalog from astropyp.phot.psf import SinglePSF # If no exposure time was passed to the stack and all of the # ccds had the same exposure time, use that to calcualte the # psf magnitude if exptime is None: if hasattr(self, 'exptime'): exptime = self.exptime else: same_exptime = True exptime = np.nan for ccd in self.ccds: if np.isnan(exptime): exptime = ccd.exptime elif ccd.exptime != exptime: same_exptime = False if not same_exptime: exptime = None if hasattr(self.catalog, 'peak'): peak = self.catalog.peak else: from astropyp.utils.misc import update_ma_idx peak = [ update_ma_idx(self.ccds[n].catalog['peak'], self.catalog['idx_'+str(n)]) for n in self.ccd_indices] peak = np.ma.mean(peak, axis=0) # Get the positions and estimated amplitudes of # the sources to fit if indices is not None: positions = zip(self.catalog.x[indices], self.catalog.y[indices]) amplitudes = peak[indices] else: positions = zip(self.catalog.x, self.catalog.y) amplitudes = peak src_count = len(positions) src_indices = np.arange(0,len(self.catalog.sources),1) all_positions = np.array(zip(self.catalog.x, self.catalog.y)) all_amplitudes = peak total_sources = len(all_amplitudes) src_psfs = [] psf_flux = np.ones((total_sources,))*np.nan psf_flux_err = np.ones((total_sources,))*np.nan psf_x = np.ones((total_sources,))*np.nan psf_y = np.ones((total_sources,))*np.nan new_amplitudes = np.ones((total_sources,))*np.nan # Find nearest neighbors to avoid contamination by nearby sources if separation is not None: if not hasattr(self, 'kdtree'): from scipy import spatial KDTree = spatial.cKDTree self.kd_tree = KDTree(all_positions) idx, nidx = astropyp.catalog.find_neighbors(separation, kd_tree=self.kd_tree) # Get parameters to pass to the pool manager imgs = [] dqmasks = [] tx_solutions = [] for n,ccd in enumerate(self.ccds): imgs.append(ccd.img) dqmasks.append(ccd.dqmask) if n!= self.ref_index: tx_solutions.append(self.tx_solutions[(self.ref_index, n)]) else: tx_solutions.append(None) if pool_size is None: pool_size = multiprocessing.cpu_count() # Create a pool with the static (for all sources) # variables to speed up processing pool = multiprocessing.Pool( processes=pool_size, initializer=_init_multiprocess, initargs=(imgs, dqmasks, tx_solutions, self.psf.shape, self.ref_index, stack_method, self.subsampling, self.psf)) pool_args = [] # Fit each source to the PSF, calcualte its flux, and its new # position for n in range(len(positions)): if indices is not None: src_idx = src_indices[indices][n] else: src_idx = src_indices[n] if separation is not None: n_indices = nidx[idx==src_idx] neighbor_positions = all_positions[n_indices] neighbor_amplitudes = all_amplitudes[n_indices] else: neighbor_positions = [] neighbor_amplitudes = [] pool_args.append((amplitudes[n], (positions[n][1], positions[n][0]), self.subsampling, neighbor_positions, neighbor_amplitudes)) result = pool.map(_stack_psf_worker, pool_args) pool.close() pool.join() # Update the various psf array parameters for n in range(len(positions)): if indices is not None: src_idx = src_indices[indices][n] else: src_idx = src_indices[n] psf_flux[src_idx] = result[n][0] psf_flux_err[src_idx] = result[n][1] new_amplitudes[src_idx] = result[n][2] psf_x[src_idx], psf_y[src_idx] = result[n][3] # Save the psf derived quantities in the catalog # Ignore divide by zero errors that occur when sources # have zero psf flux (i.e. bad sources) if save_catalog: np_err = np.geterr() np.seterr(divide='ignore') self.catalog.sources['psf_flux'] = psf_flux self.catalog.sources['psf_flux_err'] = psf_flux_err if exptime is not None: psf_mag = -2.5*np.log10(psf_flux/exptime) self.catalog.sources['psf_mag'] = psf_mag self.catalog.sources['psf_mag_err'] = psf_flux_err/psf_flux self.catalog.sources['psf_x'] = psf_x self.catalog.sources['psf_y'] = psf_y np.seterr(**np_err) return psf_flux, psf_flux_err
def get_all_merged_indices(all_coord1, all_coord2, pool_size=None, separation=1/3600., merge_type='outer'): """ Get masked indices for a set of ra,dec that merge the sources together using an outer join Parameters ---------- all_coord1: list of array-like List of arrays of values for the first coordinate (usually RA or X) all_coord2: list of array-like List of arrays of values for the second coordinate (usually DEC or Y) pool_size: int, optional Number of processors to use to match coordinates. If ``pool_size is None`` (default) then the maximum number of processors is used. separation: float, optional Maximum distance between two coordinates for a match. The default is ``1/3600``, or 1 arcsec. merge_type: str, optional Type of merge to use. This must be 'outer','inner', 'left', or 'right'. The default is 'outer'. Returns ------- indices: list of masked arrays Indices to match each catalog in all_coord1, all_coord2 to the master catalog matched: array Indices of rows that has an entry for *every* set of coordinates all_duplicates: array Indices of rows that have duplicate values mean_coord1: array Average coord1 for each row mean_coord2: array Average coord2 for each row """ from astropyp.utils import misc if merge_type not in ['outer','inner','left','right']: raise ValueError( "merge_type must be 'outer','inner', 'left', or 'right'") # Initialize indices and coordinates indices = [np.ma.array([n for n in range(all_coord1[m].shape[0])], dtype=int) for m in range(len(all_coord1))] mean_coord1 = np.ma.array(all_coord1[0]) mean_coord2 = np.ma.array(all_coord2[0]) all_duplicates = np.zeros(mean_coord1.shape, dtype=bool) # Create merged indices for n in range(1,len(all_coord1)): idx0, idx1, duplicates = get_merged_indices( (all_coord1[n],all_coord2[n]), ref_coords=(mean_coord1,mean_coord2), pool_size=pool_size, separation=separation) new_idx, new_unmatched = idx0 ref_idx, ref_unmatched = idx1 # Update list of duplicates duplicates_unmatched = all_duplicates[ref_unmatched] all_duplicates = all_duplicates[ref_idx] all_duplicates[duplicates] = True # Update indices if merge_type=='outer' or merge_type=='left': ref_idx = np.hstack([ref_idx, ref_unmatched]) new_idx = np.hstack([new_idx, -np.ones(ref_unmatched.shape, dtype=int)]) all_duplicates = np.hstack([all_duplicates, duplicates_unmatched]) if merge_type=='outer' or merge_type=='right': ref_idx = np.hstack([ref_idx, -np.ones(new_unmatched.shape, dtype=int)]) new_idx = np.hstack([new_idx, new_unmatched]) all_duplicates = np.hstack([all_duplicates, np.zeros(new_unmatched.shape, dtype=bool)]) # Mask indices ref_mask = ref_idx<0 new_mask = new_idx<0 ref_idx = np.ma.array(ref_idx, mask=ref_mask) new_idx = np.ma.array(new_idx, mask=new_mask) # Update the mean coordinate values mean_coord1 = misc.update_ma_idx(mean_coord1,ref_idx) mean_coord2 = misc.update_ma_idx(mean_coord2,ref_idx) new_coord1 = misc.update_ma_idx(all_coord1[n],new_idx) new_coord2 = misc.update_ma_idx(all_coord2[n],new_idx) mean_coord1 = np.ma.mean( np.ma.vstack([mean_coord1, new_coord1]), axis=0) mean_coord2 = np.ma.mean( np.ma.vstack([mean_coord2, new_coord2]), axis=0) # Update all of the indices with the new matches for m in range(n): indices[m] = misc.update_ma_idx(indices[m],ref_idx) indices[n] = new_idx matched = np.sum([i.mask for i in indices],axis=0)==0 return indices, matched, all_duplicates, mean_coord1, mean_coord2