Ejemplo n.º 1
0
def find_sources_with_sep(img):
    """Return sources (x, y) sorted by brightness. Use SEP package.
    """
    import sep
    if isinstance(img, np.ma.MaskedArray):
        image = img.filled(fill_value=np.median(img)).astype('float32')
    else:
        image = img.astype('float32')

    bkg = sep.Background(image)
    thresh = 3. * bkg.globalrms
    try:
        sources = sep.extract(image - bkg.back(), thresh)
    except Exception as e:
        buff_message = 'internal pixel buffer full'
        if e.message[0:26] == buff_message:
            sep.set_extract_pixstack(600000)
        try:
            sources = sep.extract(image - bkg.back(), thresh)
        except Exception as e:
            if e.message[0:26] == buff_message:
                sep.set_extract_pixstack(900000)
                sources = sep.extract(image - bkg.back(), thresh)

    sources.sort(order='flux')
    return np.array([[asrc['x'], asrc['y']] for asrc in sources[::-1]])
Ejemplo n.º 2
0
Archivo: test.py Proyecto: cmccully/sep
def test_set_pixstack():
    """Ensure that setting the pixel stack size works."""
    old = sep.get_extract_pixstack()
    new = old * 2
    sep.set_extract_pixstack(new)
    assert new == sep.get_extract_pixstack()
    sep.set_extract_pixstack(old)
Ejemplo n.º 3
0
    def run_sep_extractor(self):
        # Not entirely sure what this is for but the likeliness of a memory
        # error occurring in call to sep.extract is inversely proportional to
        # the pixstack...
        sep.set_extract_pixstack(10000000)

        data = self.image_data_formatted
        # generate a background map of data
        bkg = sep.Background(data)
        # subtract the background map from data
        data_sub = data - bkg
        self.data_sub_bkg = data_sub
        threshold = 100  #2 * np.std(data_sub) + np.min(data_sub)
        star_objects = sep.extract(data_sub,
                                   threshold,
                                   minarea=9,
                                   mask=self.image_mask,
                                   gain=3,
                                   deblend_nthresh=32,
                                   deblend_cont=0.0005)

        pix_thresh = 100
        good_objects = star_objects[star_objects['flag'] < 8]
        good_objects = good_objects[good_objects['npix'] < pix_thresh]

        return good_objects
Ejemplo n.º 4
0
def find_sources_with_sep(img):
    """Return sources (x, y) sorted by brightness. Use SEP package.
    """
    import sep
    if isinstance(img, np.ma.MaskedArray):
        image = img.filled(fill_value=np.median(img)).astype('float32')
    else:
        image = img.astype('float32')

    bkg = sep.Background(image)
    thresh = 3. * bkg.globalrms
    try:
        sources = sep.extract(image - bkg.back(), thresh)
    except Exception as e:
        buff_message = 'internal pixel buffer full'
        if e.message[0:26] == buff_message:
            sep.set_extract_pixstack(600000)
        try:
            sources = sep.extract(image - bkg.back(), thresh)
        except Exception as e:
            if e.message[0:26] == buff_message:
                sep.set_extract_pixstack(900000)
                sources = sep.extract(image - bkg.back(), thresh)

    sources.sort(order='flux')
    return np.array([[asrc['x'], asrc['y']] for asrc in sources[::-1]])
Ejemplo n.º 5
0
    def __init__(self, config_fn=None, config={}):

        if config_fn is not None:
            config = utils.read_config(config_fn)

        self.extract_pixstack = config.pop('extract_pixstack', 300000)
        sep.set_extract_pixstack(self.extract_pixstack)
        self.step_kws = config
        self.sources = {}
Ejemplo n.º 6
0
    def __init__(self, config_fn=None, config={}, log_level='info'):

        logger._set_defaults(level=log_level.upper())

        if config_fn is not None:
            config = utils.read_config(config_fn)

        self.extract_pixstack = config.pop('extract_pixstack', 300000)
        sep.set_extract_pixstack(self.extract_pixstack)

        self.config = config
        self.sources = {}
Ejemplo n.º 7
0
Archivo: test.py Proyecto: cmccully/sep
def test_long_error_msg():
    """Ensure that the error message is created successfully when
    there is an error detail."""

    # set extract pixstack to an insanely small value; this will trigger
    # a detailed error message when running sep.extract()
    old = sep.get_extract_pixstack()
    sep.set_extract_pixstack(5)

    data = np.ones((10, 10), dtype=np.float64)
    with pytest.raises(Exception) as excinfo:
        sep.extract(data, 0.1)
    msg = excinfo.value.args[0]
    assert type(msg) == str  # check that message is the native string type
    assert msg.startswith("internal pixel buffer full: The limit")

    # restore
    sep.set_extract_pixstack(old)
Ejemplo n.º 8
0
def run_sep(data, header, background_mask_threshold=None):
    """Compute mask, background, err; then sep.extract sources.
    """
    extract_SN_threshold = config['SEP']['extract_SN_threshold']
    background_mask_threshold_scale_factor = config['SEP']['background_mask_threshold_scale_factor']
    bg_box_size = config['SEP']['bg_box_size']
    min_area = config['SEP']['min_area']

    # Pupils cover more pixels than point sources.
    # So, set the number of source pixels to be 10% of the total (default=300000 pixels)
    sep.set_extract_pixstack(max(300000, int(data.size * 0.1)))

    if background_mask_threshold is None:
        background_mask_threshold = background_mask_threshold_scale_factor * np.sqrt(np.median(data)) + np.median(data)

    background = sep.Background(np.ascontiguousarray(data),
                                mask=np.ascontiguousarray(data > background_mask_threshold),
                                bw=bg_box_size, bh=bg_box_size)

    read_noise_e = float(header['RDNOISE'])
    extract_err = np.sqrt(data + read_noise_e ** 2.0)
    return sep.extract(data - background, extract_SN_threshold,
                       err=extract_err, minarea=min_area, deblend_cont=1.0, filter_kernel=None)
Ejemplo n.º 9
0
    def _best_srcs(self):
        """Property, a dictionary of best sources detected in the image.
        Keys are:
            fitshape: tuple, the size of the stamps on each source detected
            sources: a table of sources, with the imformation from sep
            positions: an array, with the position of each source stamp
            n_sources: the total number of sources extracted
        """
        if not hasattr(self, '_best_sources'):
            try:
                srcs = sep.extract(self.bkg_sub_img,
                                   thresh=6 * self.bkg.globalrms,
                                   mask=self.masked.mask)
            except Exception:
                sep.set_extract_pixstack(700000)
                srcs = sep.extract(self.bkg_sub_img,
                                   thresh=8 * self.bkg.globalrms,
                                   mask=self.masked.mask)
            except ValueError:
                srcs = sep.extract(self.bkg_sub_img.byteswap().newbyteorder(),
                                   thresh=8 * self.bkg.globalrms,
                                   mask=self.masked.mask)

            if len(srcs) < 20:
                try:
                    srcs = sep.extract(self.bkg_sub_img,
                                       thresh=5 * self.bkg.globalrms,
                                       mask=self.masked.mask)
                except Exception:
                    sep.set_extract_pixstack(900000)
                    srcs = sep.extract(self.bkg_sub_img,
                                       thresh=5 * self.bkg.globalrms,
                                       mask=self.masked.mask)
            if len(srcs) < 10:
                print 'No sources detected'

            #~ print 'raw sources = {}'.format(len(srcs))

            p_sizes = np.percentile(srcs['npix'], q=[25, 55, 75])

            best_big = srcs['npix'] >= p_sizes[0]
            best_small = srcs['npix'] <= p_sizes[2]
            best_flag = srcs['flag'] <= 1
            fluxes_quartiles = np.percentile(srcs['flux'], q=[15, 85])
            low_flux = srcs['flux'] > fluxes_quartiles[0]
            # hig_flux = srcs['flux'] < fluxes_quartiles[1]

            # best_srcs = srcs[best_big & best_flag & best_small & hig_flux & low_flux]
            best_srcs = srcs[best_flag & best_small & low_flux & best_big]

            if self._shape is not None:
                fitshape = self._shape
            else:
                p_sizes = 3. * np.sqrt(
                    np.percentile(best_srcs['npix'], q=[35, 65, 95]))

                if p_sizes[1] >= 21:
                    dx = int(p_sizes[1])
                    if dx % 2 != 1: dx += 1
                    fitshape = (dx, dx)
                else:
                    fitshape = (21, 21)

            if len(best_srcs) > 1800:
                jj = np.random.choice(len(best_srcs), 1800, replace=False)
                best_srcs = best_srcs[jj]

            print 'Sources good to calculate = {}'.format(len(best_srcs))
            self._best_sources = {'sources': best_srcs, 'fitshape': fitshape}

            self.db = npdb.NumPyDB_cPickle(self.dbname, mode='store')

            pos = []
            jj = 0
            for row in best_srcs:
                position = [row['y'], row['x']]
                sub_array_data = extract_array(self.bkg_sub_img,
                                               fitshape,
                                               position,
                                               fill_value=self.bkg.globalrms)
                sub_array_data = sub_array_data / np.sum(sub_array_data)

                # Patch.append(sub_array_data)
                self.db.dump(sub_array_data, jj)
                pos.append(position)
                jj += 1

            # self._best_sources['patches'] = np.array(Patch)
            self._best_sources['positions'] = np.array(pos)
            self._best_sources['n_sources'] = jj
            # self._best_sources['detected'] = srcs
            # self.db = npdb.NumPyDB_cPickle(self._dbname, mode='store')

            #~ print 'returning best sources\n'
        return self._best_sources
Ejemplo n.º 10
0
    def do_stage(self, images):
        for i, image in enumerate(images):
            try:
                # Set the number of source pixels to be 5% of the total. This keeps us safe from
                # satellites and airplanes.
                sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))

                data = image.data.copy()
                error = (np.abs(data) + image.readnoise**2.0)**0.5
                mask = image.bpm > 0

                # Fits can be backwards byte order, so fix that if need be and subtract
                # the background
                try:
                    bkg = sep.Background(data,
                                         mask=mask,
                                         bw=32,
                                         bh=32,
                                         fw=3,
                                         fh=3)
                except ValueError:
                    data = data.byteswap(True).newbyteorder()
                    bkg = sep.Background(data,
                                         mask=mask,
                                         bw=32,
                                         bh=32,
                                         fw=3,
                                         fh=3)
                bkg.subfrom(data)

                # Do an initial source detection
                # TODO: Add back in masking after we are sure SEP works
                sources = sep.extract(data,
                                      self.threshold,
                                      minarea=self.min_area,
                                      err=error,
                                      deblend_cont=0.005)

                # Convert the detections into a table
                sources = Table(sources)

                # Calculate the ellipticity
                sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])

                # Fix any value of theta that are invalid due to floating point rounding
                # -pi / 2 < theta < pi / 2
                sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
                sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi

                # Calculate the kron radius
                kronrad, krflag = sep.kron_radius(data, sources['x'],
                                                  sources['y'], sources['a'],
                                                  sources['b'],
                                                  sources['theta'], 6.0)
                sources['flag'] |= krflag
                sources['kronrad'] = kronrad

                # Calcuate the equivilent of flux_auto
                flux, fluxerr, flag = sep.sum_ellipse(data,
                                                      sources['x'],
                                                      sources['y'],
                                                      sources['a'],
                                                      sources['b'],
                                                      np.pi / 2.0,
                                                      2.5 * kronrad,
                                                      subpix=1,
                                                      err=error)
                sources['flux'] = flux
                sources['fluxerr'] = fluxerr
                sources['flag'] |= flag

                # Calculate the FWHMs of the stars:
                fwhm = 2.0 * (np.log(2) *
                              (sources['a']**2.0 + sources['b']**2.0))**0.5
                sources['fwhm'] = fwhm

                # Cut individual bright pixels. Often cosmic rays
                sources = sources[fwhm > 1.0]

                # Measure the flux profile
                flux_radii, flag = sep.flux_radius(data,
                                                   sources['x'],
                                                   sources['y'],
                                                   6.0 * sources['a'],
                                                   [0.25, 0.5, 0.75],
                                                   normflux=sources['flux'],
                                                   subpix=5)
                sources['flag'] |= flag
                sources['fluxrad25'] = flux_radii[:, 0]
                sources['fluxrad50'] = flux_radii[:, 1]
                sources['fluxrad75'] = flux_radii[:, 2]

                # Calculate the windowed positions
                sig = 2.0 / 2.35 * sources['fluxrad50']
                xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
                                              sig)
                sources['flag'] |= flag
                sources['xwin'] = xwin
                sources['ywin'] = ywin

                # Calculate the average background at each source
                bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
                                                         sources['x'],
                                                         sources['y'],
                                                         sources['a'],
                                                         sources['b'],
                                                         np.pi / 2.0,
                                                         2.5 *
                                                         sources['kronrad'],
                                                         subpix=1)
                #masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
                #                                         sources['a'], sources['b'], np.pi / 2.0,
                #                                         2.5 * kronrad, subpix=1)

                background_area = (
                    2.5 * sources['kronrad']
                )**2.0 * sources['a'] * sources['b'] * np.pi  # - masksum
                sources['background'] = bkgflux
                sources['background'][background_area > 0] /= background_area[
                    background_area > 0]
                # Update the catalog to match fits convention instead of python array convention
                sources['x'] += 1.0
                sources['y'] += 1.0

                sources['xpeak'] += 1
                sources['ypeak'] += 1

                sources['xwin'] += 1.0
                sources['ywin'] += 1.0

                sources['theta'] = np.degrees(sources['theta'])

                image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak',
                                        'ypeak', 'flux', 'fluxerr',
                                        'background', 'fwhm', 'a', 'b',
                                        'theta', 'kronrad', 'ellipticity',
                                        'fluxrad25', 'fluxrad50', 'fluxrad75',
                                        'x2', 'y2', 'xy', 'flag']

                # Add the units and description to the catalogs
                image.catalog['x'].unit = 'pixel'
                image.catalog['x'].description = 'X coordinate of the object'
                image.catalog['y'].unit = 'pixel'
                image.catalog['y'].description = 'Y coordinate of the object'
                image.catalog['xwin'].unit = 'pixel'
                image.catalog[
                    'xwin'].description = 'Windowed X coordinate of the object'
                image.catalog['ywin'].unit = 'pixel'
                image.catalog[
                    'ywin'].description = 'Windowed Y coordinate of the object'
                image.catalog['xpeak'].unit = 'pixel'
                image.catalog['xpeak'].description = 'X coordinate of the peak'
                image.catalog['ypeak'].unit = 'pixel'
                image.catalog[
                    'ypeak'].description = 'Windowed Y coordinate of the peak'
                image.catalog['flux'].unit = 'counts'
                image.catalog[
                    'flux'].description = 'Flux within a Kron-like elliptical aperture'
                image.catalog['fluxerr'].unit = 'counts'
                image.catalog[
                    'fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
                image.catalog['background'].unit = 'counts'
                image.catalog[
                    'background'].description = 'Average background value in the aperture'
                image.catalog['fwhm'].unit = 'pixel'
                image.catalog['fwhm'].description = 'FWHM of the object'
                image.catalog['a'].unit = 'pixel'
                image.catalog[
                    'a'].description = 'Semi-major axis of the object'
                image.catalog['b'].unit = 'pixel'
                image.catalog[
                    'b'].description = 'Semi-minor axis of the object'
                image.catalog['theta'].unit = 'degrees'
                image.catalog[
                    'theta'].description = 'Position angle of the object'
                image.catalog['kronrad'].unit = 'pixel'
                image.catalog[
                    'kronrad'].description = 'Kron radius used for extraction'
                image.catalog['ellipticity'].description = 'Ellipticity'
                image.catalog['fluxrad25'].unit = 'pixel'
                image.catalog[
                    'fluxrad25'].description = 'Radius containing 25% of the flux'
                image.catalog['fluxrad50'].unit = 'pixel'
                image.catalog[
                    'fluxrad50'].description = 'Radius containing 50% of the flux'
                image.catalog['fluxrad75'].unit = 'pixel'
                image.catalog[
                    'fluxrad75'].description = 'Radius containing 75% of the flux'
                image.catalog['x2'].unit = 'pixel^2'
                image.catalog[
                    'x2'].description = 'Variance on X coordinate of the object'
                image.catalog['y2'].unit = 'pixel^2'
                image.catalog[
                    'y2'].description = 'Variance on Y coordinate of the object'
                image.catalog['xy'].unit = 'pixel^2'
                image.catalog['xy'].description = 'XY covariance of the object'
                image.catalog[
                    'flag'].description = 'Bit mask combination of extraction and photometry flags'

                image.catalog.sort('flux')
                image.catalog.reverse()

                logging_tags = logs.image_config_to_tags(
                    image, self.group_by_keywords)
                logs.add_tag(logging_tags, 'filename',
                             os.path.basename(image.filename))

                # Save some background statistics in the header
                mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
                image.header['L1MEAN'] = (
                    mean_background,
                    '[counts] Sigma clipped mean of frame background')
                logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))

                median_background = np.median(bkg.back())
                image.header['L1MEDIAN'] = (
                    median_background, '[counts] Median of frame background')
                logs.add_tag(logging_tags, 'L1MEDIAN',
                             float(median_background))

                std_background = stats.robust_standard_deviation(bkg.back())
                image.header['L1SIGMA'] = (
                    std_background,
                    '[counts] Robust std dev of frame background')
                logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))

                # Save some image statistics to the header
                good_objects = image.catalog['flag'] == 0

                seeing = np.median(
                    image.catalog['fwhm'][good_objects]) * image.pixel_scale
                image.header['L1FWHM'] = (seeing,
                                          '[arcsec] Frame FWHM in arcsec')
                logs.add_tag(logging_tags, 'L1FWHM', float(seeing))

                mean_ellipticity = stats.sigma_clipped_mean(
                    sources['ellipticity'][good_objects], 3.0)
                image.header['L1ELLIP'] = (mean_ellipticity,
                                           'Mean image ellipticity (1-B/A)')
                logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))

                mean_position_angle = stats.sigma_clipped_mean(
                    sources['theta'][good_objects], 3.0)
                image.header['L1ELLIPA'] = (
                    mean_position_angle, '[deg] PA of mean image ellipticity')
                logs.add_tag(logging_tags, 'L1ELLIPA',
                             float(mean_position_angle))

                self.logger.info('Extracted sources', extra=logging_tags)

            except Exception as e:
                logging_tags = logs.image_config_to_tags(
                    image, self.group_by_keywords)
                logs.add_tag(logging_tags, 'filename',
                             os.path.basename(image.filename))
                self.logger.error(e, extra=logging_tags)
        return images
Ejemplo n.º 11
0
def detect_blobs(image, mask, threshold, min_area, deblend_nthresh=500,
        deblend_cont=0):
    '''Detects blobs in `image` using an implementation of
    `SExtractor <http://www.astromatic.net/software/sextractor>`_ [1].

    Parameters
    ----------
    image: numpy.ndarray[Union[numpy.uint8, numpy.uint16]]
        grayscale image in which blobs should be detected
    mask: numpy.ndarray[numpy.bool]
        binary image that masks pixel regions in which no blobs should be
        detected
    threshold: int, optional
        factor by which pixel values must be above background
        to be considered part of a blob (default: ``5``)
    min_area: int, optional
        minimal size of a blob
    deblend_ntresh: int, optional
        number of deblending thresholds (default: ``500``)
    deblend_cont: int, optional
        minimum contrast ratio for deblending (default: ``0``)

    Returns
    -------
    Tuple[numpy.ndarray[numpy.int32]]
        detected blobs and the corresponding centroids

    References
    ----------
    .. [1] Bertin, E. & Arnouts, S. 1996: SExtractor: Software for source
    extraction, Astronomy & Astrophysics Supplement 317, 393
    '''
    sep.set_extract_pixstack(10**7)

    img = image.astype('float')

    # We pad the image with mirrored pixels to prevent border artifacts.
    pad = 50
    left = img[:, 1:pad]
    right = img[:, -pad:-1]
    detect_img = np.c_[np.fliplr(left), img, np.fliplr(right)]
    upper = detect_img[1:pad, :]
    lower = detect_img[-pad:-1, :]
    detect_img = np.r_[np.flipud(upper), detect_img, np.flipud(lower)]

    logger.info('detect blobs via thresholding and deblending')
    detection, blobs = sep.extract(
        detect_img, threshold,
        minarea=min_area, segmentation_map=True,
        deblend_nthresh=deblend_nthresh, deblend_cont=deblend_cont,
        filter_kernel=None, clean=False
    )

    centroids = np.zeros(detect_img.shape, dtype=np.int32)
    y = detection['y'].astype(int)
    x = detection['x'].astype(int)
    # WTF? In rare cases object coorindates lie outside of the image.
    n = len(detection)
    y[y > detect_img.shape[0]] = detect_img.shape[0]
    x[x > detect_img.shape[1]] = detect_img.shape[1]
    centroids[y, x] = np.arange(1, n + 1)

    # Remove the padded border pixels
    blobs = blobs[pad-1:-(pad-1), pad-1:-(pad-1)].copy()
    centroids = centroids[pad-1:-(pad-1), pad-1:-(pad-1)].copy()

    # Blobs detected outside of regions of interest are discarded.
    blobs[mask > 0] = 0
    blobs[mh.bwperim(np.invert(mask)) > 0] = 0
    mh.labeled.relabel(blobs, inplace=True)

    # We need to ensure that centroids are labeled the same way as blobs.
    centroids[centroids > 0] = blobs[centroids > 0]

    return (blobs, centroids)
Ejemplo n.º 12
0
def join_images(science_raw, science_mask, reference_raw, reference_mask, sigma_cut=5., use_pixels=False, show=False,
                percent=99, size_cut=True, pixstack_limit=None):
    """Join two images to fittable vectors"""

    science = np.ma.array(science_raw, mask=science_mask, copy=True)
    reference = np.ma.array(reference_raw, mask=reference_mask, copy=True)
    science_std, _ = fit_noise(science)
    reference_std, _ = fit_noise(reference)
    if use_pixels:
        # remove pixels less than `percent` percentile above sky level to speed fitting
        science.mask[science <= np.nanpercentile(science.compressed(), percent)] = True
        reference.mask[reference <= np.nanpercentile(reference.compressed(), percent)] = True

        # flatten into 1d arrays of good pixels
        science.mask |= reference.mask
        reference.mask |= science.mask
        science_flatten = science.compressed()
        reference_flatten = reference.compressed()
        logging.info('Found {0} usable pixels for gain matching'.format(science_flatten.size))
        if science_flatten.size == 0:
            logging.error('No pixels in common at this percentile ({0}); lower and try again'.format(percent))
    else:
        if pixstack_limit is None:
            pixstack_limit = science.size // 20
        if pixstack_limit > 300000:
            sep.set_extract_pixstack(pixstack_limit)
        science_sources = sep.extract(np.ascontiguousarray(science.data), thresh=sigma_cut, err=science_std, mask=np.ascontiguousarray(science.mask))
        reference_sources = sep.extract(np.ascontiguousarray(reference.data), thresh=sigma_cut, err=reference_std, mask=np.ascontiguousarray(reference.mask))
        science_sources = science_sources[science_sources['errx2'] != np.inf]  # exclude partially masked sources
        reference_sources = reference_sources[reference_sources['errx2'] != np.inf]
        dx = science_sources['x'] - reference_sources['x'][:, np.newaxis]
        dy = science_sources['y'] - reference_sources['y'][:, np.newaxis]
        separation = np.sqrt(dx**2 + dy**2)
        sigma_eqv = np.sqrt((reference_sources['a']**2 + reference_sources['b']**2) / 2.)
        matches = (np.min(separation, axis=1) < 2. * sigma_eqv)
        if size_cut:
            # cut unusually large/small sources (assumes most sources are real)
            med_sigma = np.median(sigma_eqv)  # median sigma if all sources were circular Gaussians
            absdev_sigma = np.abs(sigma_eqv - med_sigma)
            std_sigma = np.median(absdev_sigma) * np.sqrt(np.pi / 2)
            matches &= (absdev_sigma < 3 * std_sigma)
        inds = np.argmin(separation, axis=1)
        science_flatten = science_sources['flux'][inds][matches]
        reference_flatten = reference_sources['flux'][matches]
        logging.info('Found {0} stars in common for gain matching'.format(science_flatten.size))
        if science_flatten.size <= 2:
            logging.error('Too few stars in common at {0}-sigma; lower and try again'.format(sigma_cut))
            raise ValueError()

    if show:
        import matplotlib.pyplot as plt

        plt.ion()
        plt.figure(1)
        plt.clf()
        vmin, vmax = np.nanpercentile(science, (1, 99))
        plt.imshow(science, vmin=vmin, vmax=vmax)
        plt.title('Science')
        if not use_pixels:
            plt.plot(reference_sources['x'][matches], reference_sources['y'][matches], 'o', mfc='none', mec='r')
        
        plt.figure(2)
        plt.clf()
        vmin, vmax = np.nanpercentile(reference, (1, 99))
        plt.imshow(reference, vmin=vmin, vmax=vmax)
        plt.title('Reference')
        if not use_pixels:
            plt.plot(reference_sources['x'][matches], reference_sources['y'][matches], 'o', mfc='none', mec='r')
        
        plt.figure(3)
        plt.clf()
        plt.loglog(reference_flatten, science_flatten, '.')
        plt.xlabel('Reference')
        plt.ylabel('Science')

    return reference_flatten, science_flatten
Ejemplo n.º 13
0
    def sextract(self,
                 band,
                 sub_background=False,
                 force_segmap=None,
                 use_mask=False,
                 incl_apphot=False):
        # perform sextractor on single band only (may expand for matched source phot)
        # Generate segmap and segmask

        if (band == conf.DETECTION_NICKNAME) | (band
                                                == conf.MODELING_NICKNAME):
            idx = 0
        else:
            idx = self._band2idx(band)
        image = self.images[idx].copy()
        wgts = self.weights[idx].copy()
        wgts[wgts == 0] = -1
        var = 1. / wgts
        var[wgts == -1] = 0
        mask = self.masks[idx].copy()
        background = self.backgrounds[idx]

        # Supply a segmap to "match" detection
        if force_segmap is not None:
            var[force_segmap] = 0

        if (self.weights == 1).all() | (conf.USE_DETECTION_WEIGHT == False):
            # No weight supplied by user
            var = None
            thresh = conf.THRESH  #* background[1]  # WTF is this about?!
            # if not sub_background:
            #     thresh += background[0]

            self.logger.debug(
                f'Detection is to be performed with weights? {conf.USE_DETECTION_WEIGHT}'
            )
        else:
            thresh = conf.THRESH

        convfilt = None
        if conf.FILTER_KERNEL is not None:
            dirname = os.path.dirname(__file__)
            filename = os.path.join(
                dirname, '../../config/conv_filters/' + conf.FILTER_KERNEL)
            if os.path.exists(filename):
                convfilt = np.array(
                    np.array(ascii.read(filename, data_start=1)).tolist())
            else:
                raise FileExistsError(
                    f"Convolution file at {filename} does not exist!")

        if use_mask:
            mask = mask
        else:
            mask = None

        self.logger.debug(
            f'Detection is to be performed with thresh = {thresh}')

        # Set extraction pixel limit buffer
        sep.set_extract_pixstack(conf.PIXSTACK_SIZE)

        if sub_background:
            self.logger.debug('Background will be subtracted.')
            background = sep.Background(self.images[idx],
                                        bw=conf.DETECT_BW,
                                        bh=conf.DETECT_BH,
                                        fw=conf.DETECT_FW,
                                        fh=conf.DETECT_FH)
            if conf.USE_FLAT:
                image -= background.globalback
                self.logger.debug(
                    f'Subtracted flat background level ({background.globalback:4.4f})'
                )
            else:
                image -= background.back()

        # var = np.ones_like(var)
        kwargs = dict(var=var,
                      mask=mask,
                      minarea=conf.MINAREA,
                      filter_kernel=convfilt,
                      filter_type=conf.FILTER_TYPE,
                      segmentation_map=True,
                      clean=conf.CLEAN,
                      clean_param=conf.CLEAN_PARAM,
                      deblend_nthresh=conf.DEBLEND_NTHRESH,
                      deblend_cont=conf.DEBLEND_CONT)
        catalog, segmap = sep.extract(image, thresh, **kwargs)
        catalog['y'] -= 0.75  # HACK
        catalog['x'] -= 0.1  #HACK

        # plt.ion()
        # fig, ax = plt.subplots(ncols=2, figsize=(40,20))
        # from matplotlib.colors import LogNorm
        # ax[0].imshow(image, norm=LogNorm(), vmin=1E-8, vmax=10, cmap='Greys')
        # ax[0].scatter(catalog['x'], catalog['y'], s=1)
        # ax[1].imshow(segmap)
        # ax[1].scatter(catalog['x'], catalog['y'], s=1)
        # # plt.show()
        # fig.savefig(conf.PLOT_DIR + '/checksep.pdf')
        # plt.close('all')

        if len(catalog) != 0:
            catalog = Table(catalog)
            catalog.add_column(Column(catalog['x'], name='x_orig'))
            catalog.add_column(Column(catalog['y'], name='y_orig'))

            if self.wcs is not None:
                wx = catalog[
                    'x_orig']  #+ self.mosaic_origin[1] - conf.BRICK_BUFFER
                wy = catalog[
                    'y_orig']  #+ self.mosaic_origin[0] - conf.BRICK_BUFFER
                wwx, wwy = wx, wy
                # wwx, wwy = wx - self.mosaic_origin[0] + conf.BRICK_BUFFER, wy - self.mosaic_origin[1] + conf.BRICK_BUFFER
                skyc = self.wcs.all_pix2world(wwx, wwy, 0)
                # print(- self.mosaic_origin[0] + conf.BRICK_BUFFER, - self.mosaic_origin[1] + conf.BRICK_BUFFER)
                catalog.add_column(
                    Column(skyc[0], name=f'RA_{conf.DETECTION_NICKNAME}'))
                catalog.add_column(
                    Column(skyc[1], name=f'DEC_{conf.DETECTION_NICKNAME}'))

            # Aperture Photometry
            if incl_apphot:
                flux = np.zeros(len(catalog),
                                dtype=(float, len(conf.APER_PHOT)))
                flux_err = flux.copy()
                flag = np.zeros_like(catalog['x'], dtype=bool)
                for i, radius in enumerate(conf.APER_PHOT):  # in arcsec
                    flux[:, i], flux_err[:, i], flag = sep.sum_circle(
                        image,
                        x=catalog['x'],
                        y=catalog['y'],
                        r=radius / conf.PIXEL_SCALE,
                        var=var)

                mag = -2.5 * np.log10(flux) + conf.MODELING_ZPT
                mag_err = 1.09 * flux_err / flux
                catalog.add_column(Column(flux, name='flux_aper'))
                catalog.add_column(Column(flux_err, name='fluxerr_aper'))
                catalog.add_column(Column(mag, name='mag_aper'))
                catalog.add_column(Column(mag_err, name='magerr_aper'))
                catalog.add_column(Column(flag, name='flag_aper'))

            self.catalog = catalog
            self.n_sources = len(catalog)
            self.segmap = segmap
            return catalog, segmap
        else:
            raise ValueError('No objects found by SExtractor.')
Ejemplo n.º 14
0
(v0_2) - January-2019 Spec2SDSS_gri now incorporates the redshift (factor of (1+z)**5) to the wavelength specific intensities when generating the bandpass AB surface brightnesses. The redshift factor is now removed from intensity scaling step in this version.
(v0_3) - January-2019 Computes ra,dec from the source mask when determining image position. This avoids image registration offsets in row and column numbers for each band. Prepared for public release.
'''

import numpy as np
import os, sys, string, time
from scipy.interpolate import RectBivariateSpline
import scipy.ndimage
import warnings
from astropy.io import fits
from astropy.utils.exceptions import AstropyWarning
from astropy.convolution import Gaussian2DKernel
from astropy.convolution import convolve
from astropy.cosmology import FlatLambdaCDM
import sep
sep.set_extract_pixstack(9999999)

realsim_dir = os.path.dirname(os.path.abspath(__file__))


def rebin(array, dimensions=None, scale=None):
    """
    Return the array 'array' to the new 'dimensions'
    conserving the flux in the bins. The sum of the
    array will remain the same as the original array.
    Congrid from the scipy recipies does not generally
    conserve surface brightness with reasonable accuracy.
    As such, even accounting for the ratio of old and new
    image areas, it does not conserve flux. This function
    nicely solves the problem and more accurately
    redistributes the flux in the output. This function
Ejemplo n.º 15
0
    def do_stage(self, image):
        try:
            # Set the number of source pixels to be 5% of the total. This keeps us safe from
            # satellites and airplanes.
            sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))

            data = image.data.copy()
            error = (np.abs(data) + image.readnoise**2.0)**0.5
            mask = image.bpm > 0

            # Fits can be backwards byte order, so fix that if need be and subtract
            # the background
            try:
                bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
            except ValueError:
                data = data.byteswap(True).newbyteorder()
                bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
            bkg.subfrom(data)

            # Do an initial source detection
            # TODO: Add back in masking after we are sure SEP works
            sources = sep.extract(data,
                                  self.threshold,
                                  minarea=self.min_area,
                                  err=error,
                                  deblend_cont=0.005)

            # Convert the detections into a table
            sources = Table(sources)

            # We remove anything with a detection flag >= 8
            # This includes memory overflows and objects that are too close the edge
            sources = sources[sources['flag'] < 8]

            sources = array_utils.prune_nans_from_table(sources)

            # Calculate the ellipticity
            sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])

            # Fix any value of theta that are invalid due to floating point rounding
            # -pi / 2 < theta < pi / 2
            sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
            sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi

            # Calculate the kron radius
            kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
                                              sources['a'], sources['b'],
                                              sources['theta'], 6.0)
            sources['flag'] |= krflag
            sources['kronrad'] = kronrad

            # Calcuate the equivilent of flux_auto
            flux, fluxerr, flag = sep.sum_ellipse(data,
                                                  sources['x'],
                                                  sources['y'],
                                                  sources['a'],
                                                  sources['b'],
                                                  np.pi / 2.0,
                                                  2.5 * kronrad,
                                                  subpix=1,
                                                  err=error)
            sources['flux'] = flux
            sources['fluxerr'] = fluxerr
            sources['flag'] |= flag

            # Do circular aperture photometry for diameters of 1" to 6"
            for diameter in [1, 2, 3, 4, 5, 6]:
                flux, fluxerr, flag = sep.sum_circle(data,
                                                     sources['x'],
                                                     sources['y'],
                                                     diameter / 2.0 /
                                                     image.pixel_scale,
                                                     gain=1.0,
                                                     err=error)
                sources['fluxaper{0}'.format(diameter)] = flux
                sources['fluxerr{0}'.format(diameter)] = fluxerr
                sources['flag'] |= flag

            # Calculate the FWHMs of the stars:
            fwhm = 2.0 * (np.log(2) *
                          (sources['a']**2.0 + sources['b']**2.0))**0.5
            sources['fwhm'] = fwhm

            # Cut individual bright pixels. Often cosmic rays
            sources = sources[fwhm > 1.0]

            # Measure the flux profile
            flux_radii, flag = sep.flux_radius(data,
                                               sources['x'],
                                               sources['y'],
                                               6.0 * sources['a'],
                                               [0.25, 0.5, 0.75],
                                               normflux=sources['flux'],
                                               subpix=5)
            sources['flag'] |= flag
            sources['fluxrad25'] = flux_radii[:, 0]
            sources['fluxrad50'] = flux_radii[:, 1]
            sources['fluxrad75'] = flux_radii[:, 2]

            # Calculate the windowed positions
            sig = 2.0 / 2.35 * sources['fluxrad50']
            xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
                                          sig)
            sources['flag'] |= flag
            sources['xwin'] = xwin
            sources['ywin'] = ywin

            # Calculate the average background at each source
            bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
                                                     sources['x'],
                                                     sources['y'],
                                                     sources['a'],
                                                     sources['b'],
                                                     np.pi / 2.0,
                                                     2.5 * sources['kronrad'],
                                                     subpix=1)
            # masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
            #                                         sources['a'], sources['b'], np.pi / 2.0,
            #                                         2.5 * kronrad, subpix=1)

            background_area = (
                2.5 * sources['kronrad']
            )**2.0 * sources['a'] * sources['b'] * np.pi  # - masksum
            sources['background'] = bkgflux
            sources['background'][background_area > 0] /= background_area[
                background_area > 0]
            # Update the catalog to match fits convention instead of python array convention
            sources['x'] += 1.0
            sources['y'] += 1.0

            sources['xpeak'] += 1
            sources['ypeak'] += 1

            sources['xwin'] += 1.0
            sources['ywin'] += 1.0

            sources['theta'] = np.degrees(sources['theta'])

            catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
                              'flux', 'fluxerr', 'peak', 'fluxaper1',
                              'fluxerr1', 'fluxaper2', 'fluxerr2', 'fluxaper3',
                              'fluxerr3', 'fluxaper4', 'fluxerr4', 'fluxaper5',
                              'fluxerr5', 'fluxaper6', 'fluxerr6',
                              'background', 'fwhm', 'a', 'b', 'theta',
                              'kronrad', 'ellipticity', 'fluxrad25',
                              'fluxrad50', 'fluxrad75', 'x2', 'y2', 'xy',
                              'flag']

            # Add the units and description to the catalogs
            catalog['x'].unit = 'pixel'
            catalog['x'].description = 'X coordinate of the object'
            catalog['y'].unit = 'pixel'
            catalog['y'].description = 'Y coordinate of the object'
            catalog['xwin'].unit = 'pixel'
            catalog['xwin'].description = 'Windowed X coordinate of the object'
            catalog['ywin'].unit = 'pixel'
            catalog['ywin'].description = 'Windowed Y coordinate of the object'
            catalog['xpeak'].unit = 'pixel'
            catalog['xpeak'].description = 'X coordinate of the peak'
            catalog['ypeak'].unit = 'pixel'
            catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
            catalog['flux'].unit = 'count'
            catalog[
                'flux'].description = 'Flux within a Kron-like elliptical aperture'
            catalog['fluxerr'].unit = 'count'
            catalog[
                'fluxerr'].description = 'Error on the flux within Kron aperture'
            catalog['peak'].unit = 'count'
            catalog['peak'].description = 'Peak flux (flux at xpeak, ypeak)'
            for diameter in [1, 2, 3, 4, 5, 6]:
                catalog['fluxaper{0}'.format(diameter)].unit = 'count'
                catalog['fluxaper{0}'.format(
                    diameter
                )].description = 'Flux from fixed circular aperture: {0}" diameter'.format(
                    diameter)
                catalog['fluxerr{0}'.format(diameter)].unit = 'count'
                catalog['fluxerr{0}'.format(
                    diameter
                )].description = 'Error on Flux from circular aperture: {0}"'.format(
                    diameter)

            catalog['background'].unit = 'count'
            catalog[
                'background'].description = 'Average background value in the aperture'
            catalog['fwhm'].unit = 'pixel'
            catalog['fwhm'].description = 'FWHM of the object'
            catalog['a'].unit = 'pixel'
            catalog['a'].description = 'Semi-major axis of the object'
            catalog['b'].unit = 'pixel'
            catalog['b'].description = 'Semi-minor axis of the object'
            catalog['theta'].unit = 'degree'
            catalog['theta'].description = 'Position angle of the object'
            catalog['kronrad'].unit = 'pixel'
            catalog['kronrad'].description = 'Kron radius used for extraction'
            catalog['ellipticity'].description = 'Ellipticity'
            catalog['fluxrad25'].unit = 'pixel'
            catalog[
                'fluxrad25'].description = 'Radius containing 25% of the flux'
            catalog['fluxrad50'].unit = 'pixel'
            catalog[
                'fluxrad50'].description = 'Radius containing 50% of the flux'
            catalog['fluxrad75'].unit = 'pixel'
            catalog[
                'fluxrad75'].description = 'Radius containing 75% of the flux'
            catalog['x2'].unit = 'pixel^2'
            catalog[
                'x2'].description = 'Variance on X coordinate of the object'
            catalog['y2'].unit = 'pixel^2'
            catalog[
                'y2'].description = 'Variance on Y coordinate of the object'
            catalog['xy'].unit = 'pixel^2'
            catalog['xy'].description = 'XY covariance of the object'
            catalog[
                'flag'].description = 'Bit mask of extraction/photometry flags'

            catalog.sort('flux')
            catalog.reverse()

            # Save some background statistics in the header
            mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
            image.header['L1MEAN'] = (
                mean_background,
                '[counts] Sigma clipped mean of frame background')

            median_background = np.median(bkg.back())
            image.header['L1MEDIAN'] = (median_background,
                                        '[counts] Median of frame background')

            std_background = stats.robust_standard_deviation(bkg.back())
            image.header['L1SIGMA'] = (
                std_background, '[counts] Robust std dev of frame background')

            # Save some image statistics to the header
            good_objects = catalog['flag'] == 0
            for quantity in ['fwhm', 'ellipticity', 'theta']:
                good_objects = np.logical_and(
                    good_objects, np.logical_not(np.isnan(catalog[quantity])))
            if good_objects.sum() == 0:
                image.header['L1FWHM'] = ('NaN',
                                          '[arcsec] Frame FWHM in arcsec')
                image.header['L1ELLIP'] = ('NaN',
                                           'Mean image ellipticity (1-B/A)')
                image.header['L1ELLIPA'] = (
                    'NaN', '[deg] PA of mean image ellipticity')
            else:
                seeing = np.median(
                    catalog['fwhm'][good_objects]) * image.pixel_scale
                image.header['L1FWHM'] = (seeing,
                                          '[arcsec] Frame FWHM in arcsec')

                mean_ellipticity = stats.sigma_clipped_mean(
                    catalog['ellipticity'][good_objects], 3.0)
                image.header['L1ELLIP'] = (mean_ellipticity,
                                           'Mean image ellipticity (1-B/A)')

                mean_position_angle = stats.sigma_clipped_mean(
                    catalog['theta'][good_objects], 3.0)
                image.header['L1ELLIPA'] = (
                    mean_position_angle, '[deg] PA of mean image ellipticity')

            logging_tags = {
                key: float(image.header[key])
                for key in [
                    'L1MEAN', 'L1MEDIAN', 'L1SIGMA', 'L1FWHM', 'L1ELLIP',
                    'L1ELLIPA'
                ]
            }

            logger.info('Extracted sources',
                        image=image,
                        extra_tags=logging_tags)
            # adding catalog (a data table) to the appropriate images attribute.
            image.data_tables['catalog'] = DataTable(data_table=catalog,
                                                     name='CAT')
        except Exception:
            logger.error(logs.format_exception(), image=image)
        return image
Ejemplo n.º 16
0
    def best_sources(self):
        """Property, a dictionary of best sources detected in the image.
        Keys are:
            fitshape: tuple, the size of the stamps on each source detected
            sources: a table of sources, with the imformation from sep
            positions: an array, with the position of each source stamp
            n_sources: the total number of sources extracted
        """
        if not hasattr(self, '_best_sources'):
            # print('looking for srcs')
            try:
                srcs = sep.extract(self.bkg_sub_img.data,
                                   thresh=8 * self.__bkg.globalrms,
                                   mask=self.mask,
                                   minarea=9)
            except Exception:
                try:
                    sep.set_extract_pixstack(3600000)
                    srcs = sep.extract(self.bkg_sub_img.data,
                                       thresh=35 * self.__bkg.globalrms,
                                       mask=self.mask,
                                       minarea=9)
                except Exception:
                    raise
            if len(srcs) < self.min_sources:
                print("""found {} sources, looking for at least {}.
                       Trying again""".format(len(srcs), self.min_sources))
                old_srcs = srcs
                try:
                    srcs = sep.extract(self.bkg_sub_img.data,
                                       thresh=3 * self.__bkg.globalrms,
                                       mask=self.mask,
                                       minarea=5)
                except Exception:
                    sep.set_extract_pixstack(900000)
                    srcs = sep.extract(self.bkg_sub_img.data,
                                       thresh=3 * self.__bkg.globalrms,
                                       mask=self.mask,
                                       minarea=9)
                if len(old_srcs) > len(srcs):
                    srcs = old_srcs

            if len(srcs) == 0:
                raise ValueError('Few sources detected on image')
            elif len(srcs) == 1:
                m, med, st = sigma_clipped_stats(
                    self.bkg_sub_img.data.flatten())  # noqa
                if st <= 0.1:
                    raise ValueError('Image is constant, possible saturated')
                if m >= 65535.:
                    raise ValueError('Image is saturated')
                else:
                    raise ValueError('only one sources. Possible saturation')

            p_sizes = np.percentile(srcs['npix'], q=[20, 50, 80])

            best_big = srcs['npix'] >= p_sizes[0]
            best_small = srcs['npix'] <= p_sizes[2]
            best_flag = srcs['flag'] == 0

            fluxes_quartiles = np.percentile(srcs['flux'], q=[15, 85])
            low_flux = srcs['flux'] >= fluxes_quartiles[0]
            hig_flux = srcs['flux'] <= fluxes_quartiles[1]

            best_srcs = srcs[best_big & best_flag & best_small & hig_flux
                             & low_flux]

            if len(best_srcs) == 0:
                print('Best sources are too few- Using everything we have!')
                best_srcs = srcs
                # raise ValueError('Few sources detected on image')

            if len(best_srcs) > 1800:
                jj = np.random.choice(len(best_srcs), 1800, replace=False)
                best_srcs = best_srcs[jj]

            print(('Sources found = {}'.format(len(best_srcs))))
            self._best_sources = best_srcs

        return self._best_sources
Ejemplo n.º 17
0
    def best_sources(self):
        """Property, a dictionary of best sources detected in the image.
        Keys are:
            fitshape: tuple, the size of the stamps on each source detected
            sources: a table of sources, with the imformation from sep
            positions: an array, with the position of each source stamp
            n_sources: the total number of sources extracted
        """
        if not hasattr(self, "_best_sources"):
            try:
                srcs = sep.extract(
                    self.bkg_sub_img.data,
                    thresh=8 * self.__bkg.globalrms,
                    mask=self.mask,
                    minarea=9,
                )
            except Exception:
                try:
                    sep.set_extract_pixstack(3600000)
                    srcs = sep.extract(
                        self.bkg_sub_img.data,
                        thresh=35 * self.__bkg.globalrms,
                        mask=self.mask,
                        minarea=9,
                    )
                except Exception:
                    raise
            if len(srcs) < self.min_sources:
                old_srcs = srcs
                try:
                    srcs = sep.extract(
                        self.bkg_sub_img.data,
                        thresh=3 * self.__bkg.globalrms,
                        mask=self.mask,
                        minarea=5,
                    )
                except Exception:
                    sep.set_extract_pixstack(900000)
                    srcs = sep.extract(
                        self.bkg_sub_img.data,
                        thresh=3 * self.__bkg.globalrms,
                        mask=self.mask,
                        minarea=9,
                    )
                if len(old_srcs) > len(srcs):
                    srcs = old_srcs

            if len(srcs) == 0:
                raise ValueError("Few sources detected on image")
            elif len(srcs) == 1:
                m, med, st = sigma_clipped_stats(
                    self.bkg_sub_img.data.flatten())  # noqa
                if st <= 0.1:
                    raise ValueError("Image is constant, possible saturated")
                if m >= 65535.0:
                    raise ValueError("Image is saturated")
                else:
                    raise ValueError("Only one source. Possible saturation")

            p_sizes = np.percentile(srcs["npix"], q=[20, 50, 80])

            best_big = srcs["npix"] >= p_sizes[0]
            best_small = srcs["npix"] <= p_sizes[2]
            best_flag = srcs["flag"] == 0

            fluxes_quartiles = np.percentile(srcs["flux"], q=[15, 85])
            low_flux = srcs["flux"] >= fluxes_quartiles[0]
            hig_flux = srcs["flux"] <= fluxes_quartiles[1]

            best_srcs = srcs[best_big & best_flag & best_small & hig_flux
                             & low_flux]

            if len(best_srcs) == 0:
                self.warning(
                    "Best sources are too few- Using everything we have!")
                best_srcs = srcs

            if len(best_srcs) > 1800:
                jj = np.random.choice(len(best_srcs), 1800, replace=False)
                best_srcs = best_srcs[jj]

            self._best_sources = best_srcs

        return self._best_sources
Ejemplo n.º 18
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import numpy as np
import sep
from astropy.io import fits
from astropy.io import ascii
from astroML import crossmatch as cx
import pandas as pd

sep.set_extract_pixstack(900000)

data_path = os.path.abspath('/home/bos0109/sersic/work/rhino/data/extract_test/stellar/CSTAR/')
images_path = os.path.join(data_path, 'images')
master_Ryan = os.path.join(images_path, 'master10_wcs.fits')

cat_path = os.path.join(data_path, 'cats/stars.dat')

cat = ascii.read(cat_path, names = ['cstarid', 'x', 'y', 'imag'])
data = fits.getdata(master_Ryan)
data = data.byteswap().newbyteorder()
bkg = sep.Background(data)
data = data - bkg

def go_test(cat, image_data, thresh):
    threshold = thresh*bkg.globalrms
    #make the extraction
    sources = sep.extract(image_data, threshold)
    cat.to_pandas()
    sources = pd.DataFrame(sources)
Ejemplo n.º 19
0
def detect_blobs(image, mask, threshold, min_area, deblend_nthresh=500,
        deblend_cont=0):
    '''Detects blobs in `image` using an implementation of
    `SExtractor <http://www.astromatic.net/software/sextractor>`_ [1].

    Parameters
    ----------
    image: numpy.ndarray[Union[numpy.uint8, numpy.uint16]]
        grayscale image in which blobs should be detected
    mask: numpy.ndarray[numpy.bool]
        binary image that masks pixel regions in which no blobs should be
        detected
    threshold: int, optional
        factor by which pixel values must be above background
        to be considered part of a blob (default: ``5``)
    min_area: int, optional
        minimal size of a blob
    deblend_ntresh: int, optional
        number of deblending thresholds (default: ``500``)
    deblend_cont: int, optional
        minimum contrast ratio for deblending (default: ``0``)

    Returns
    -------
    Tuple[numpy.ndarray[numpy.int32]]
        detected blobs and the corresponding centroids

    References
    ----------
    .. [1] Bertin, E. & Arnouts, S. 1996: SExtractor: Software for source
    extraction, Astronomy & Astrophysics Supplement 317, 393
    '''
    sep.set_extract_pixstack(10**7)

    img = image.astype('float')

    # We pad the image with mirrored pixels to prevent border artifacts.
    pad = 50
    left = img[:, 1:pad]
    right = img[:, -pad:-1]
    detect_img = np.c_[np.fliplr(left), img, np.fliplr(right)]
    upper = detect_img[1:pad, :]
    lower = detect_img[-pad:-1, :]
    detect_img = np.r_[np.flipud(upper), detect_img, np.flipud(lower)]

    logger.info('detect blobs via thresholding and deblending')
    detection, blobs = sep.extract(
        detect_img, threshold,
        minarea=min_area, segmentation_map=True,
        deblend_nthresh=deblend_nthresh, deblend_cont=deblend_cont,
        filter_kernel=None, clean=False
    )

    centroids = np.zeros(detect_img.shape, dtype=np.int32)
    y = detection['y'].astype(int)
    x = detection['x'].astype(int)
    # WTF? In rare cases object coorindates lie outside of the image.
    n = len(detection)
    y[y > detect_img.shape[0]] = detect_img.shape[0]
    x[x > detect_img.shape[1]] = detect_img.shape[1]
    centroids[y, x] = np.arange(1, n + 1)

    # Remove the padded border pixels
    blobs = blobs[pad-1:-(pad-1), pad-1:-(pad-1)].copy()
    centroids = centroids[pad-1:-(pad-1), pad-1:-(pad-1)].copy()

    # Blobs detected outside of regions of interest are discarded.
    blobs[mask > 0] = 0
    blobs[mh.bwperim(np.invert(mask)) > 0] = 0
    mh.labeled.relabel(blobs, inplace=True)

    # We need to ensure that centroids are labeled the same way as blobs.
    centroids[centroids > 0] = blobs[centroids > 0]

    return (blobs, centroids)
Ejemplo n.º 20
0
def get_objects_sep(image, header=None, mask=None, thresh=4.0, aper=3.0, bkgann=None, r0=0.5, gain=1, edge=0, minnthresh=2, minarea=5, relfluxradius=2.0, wcs=None, use_fwhm=False, use_mask_bg=False, use_mask_large=False, subtract_bg=True, npix_large=300, sn=10.0, verbose=True, get_fwhm=False, **kwargs):
    if r0 > 0.0:
        kernel = make_kernel(r0)
    else:
        kernel = None

    if verbose:
        print("Preparing background mask")

    if mask is None:
        mask = np.zeros_like(image, dtype=np.bool)

    mask_bg = np.zeros_like(mask)
    mask_segm = np.zeros_like(mask)

    if use_mask_bg:
        # Simple heuristics to mask regions with rapidly varying background
        if verbose:
            print("Masking rapidly changing background")

        bg2 = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64)

        for _ in xrange(3):
            bg1 = sep.Background(image, mask=mask|mask_bg, bw=256, bh=256)

            ibg = bg2.back() - bg1.back()

            tmp = np.abs(ibg - np.median(ibg)) > 5.0*mad_std(ibg)
            mask_bg |= dilate(tmp, np.ones([100, 100]))

        mask_bg = dilate(tmp, np.ones([100, 100]))

    if verbose:
        print("Building background map")

    bg = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64)
    if subtract_bg:
        image1 = image - bg.back()
    else:
        image1 = image.copy()

    sep.set_extract_pixstack(image.shape[0]*image.shape[1])

    if use_mask_large:
        # Mask regions around huge objects as they are most probably corrupted by saturation and blooming
        if verbose:
            print("Extracting initial objects")

        obj0,segm = sep.extract(image1, err=bg.rms(), thresh=thresh, minarea=minarea, mask=mask|mask_bg, filter_kernel=kernel, segmentation_map=True, **kwargs)

        if verbose:
            print("Dilating large objects")

        mask_segm = np.isin(segm, [_+1 for _,npix in enumerate(obj0['npix']) if npix > npix_large])
        mask_segm = dilate(mask_segm, np.ones([50, 50]))

    if verbose:
        print("Extracting final objects")

    obj0 = sep.extract(image1, err=bg.rms(), thresh=thresh, minarea=minarea, mask=mask|mask_bg|mask_segm, filter_kernel=kernel, **kwargs)

    if use_fwhm:
        # Estimate FHWM and use it to get optimal aperture size
        idx = obj0['flag'] == 0
        fwhm = 2.0*np.sqrt(np.hypot(obj0['a'][idx], obj0['b'][idx])*np.log(2))
        fwhm = 2.0*sep.flux_radius(image1, obj0['x'][idx], obj0['y'][idx], relfluxradius*fwhm*np.ones_like(obj0['x'][idx]), 0.5, mask=mask)[0]
        fwhm = np.median(fwhm)

        aper = max(1.5*fwhm, aper)

        if verbose:
            print("FWHM = %.2g, aperture = %.2g" % (fwhm, aper))

    # Windowed positional parameters are often biased in crowded fields, let's avoid them for now
    # xwin,ywin,flag = sep.winpos(image1, obj0['x'], obj0['y'], 0.5, mask=mask)
    xwin,ywin = obj0['x'], obj0['y']

    # Filter out objects too close to frame edges
    idx = (np.round(xwin) > edge) & (np.round(ywin) > edge) & (np.round(xwin) < image.shape[1]-edge) & (np.round(ywin) < image.shape[0]-edge) # & (obj0['flag'] == 0)

    if minnthresh:
        idx &= (obj0['tnpix'] >= minnthresh)

    if verbose:
        print("Measuring final objects")

    flux,fluxerr,flag = sep.sum_circle(image1, xwin[idx], ywin[idx], aper, err=bg.rms(), gain=gain, mask=mask|mask_bg|mask_segm, bkgann=bkgann)
    # For debug purposes, let's make also the same aperture photometry on the background map
    bgflux,bgfluxerr,bgflag = sep.sum_circle(bg.back(), xwin[idx], ywin[idx], aper, err=bg.rms(), gain=gain, mask=mask|mask_bg|mask_segm)

    bg = bgflux/np.pi/aper**2

    # Fluxes to magnitudes
    mag,magerr = np.zeros_like(flux), np.zeros_like(flux)
    mag[flux>0] = -2.5*np.log10(flux[flux>0])
    # magerr[flux>0] = 2.5*np.log10(1.0 + fluxerr[flux>0]/flux[flux>0])
    magerr[flux>0] = 2.5/np.log(10)*fluxerr[flux>0]/flux[flux>0]

    # better FWHM estimation - FWHM=HFD for Gaussian
    if get_fwhm:
        fwhm = 2.0*sep.flux_radius(image1, xwin[idx], ywin[idx], relfluxradius*aper*np.ones_like(xwin[idx]), 0.5, mask=mask)[0]
    else:
        fwhm = np.zeros_like(xwin[idx])

    flag |= obj0['flag'][idx]

    # Quality cuts
    fidx = (flux > 0) & (magerr < 1.0/sn)

    if wcs is None and header is not None:
        # If header is provided, we may build WCS from it
        wcs = WCS(header)

    if wcs is not None:
        # If WCS is provided we may convert x,y to ra,dec
        ra,dec = wcs.all_pix2world(obj0['x'][idx], obj0['y'][idx], 0)
    else:
        ra,dec = np.zeros_like(obj0['x'][idx]),np.zeros_like(obj0['y'][idx])

    if verbose:
        print("All done")

    return {'x':xwin[idx][fidx], 'y':ywin[idx][fidx], 'flux':flux[fidx], 'fluxerr':fluxerr[fidx], 'mag':mag[fidx], 'magerr':magerr[fidx], 'flags':obj0['flag'][idx][fidx]|flag[fidx], 'ra':ra[fidx], 'dec':dec[fidx], 'bg':bg[fidx], 'fwhm':fwhm[fidx], 'aper':aper, 'bkgann':bkgann, 'a':obj0['a'][idx][fidx], 'b':obj0['b'][idx][fidx], 'theta':obj0['theta'][idx][fidx]}
Ejemplo n.º 21
0
s = raw_input('type glob pattern for file list: ')
flist = glob.glob(s)
print flist

#creates an empty file to fill with opened input data
inptarray = []

#Open the images given at the command line and place them in an array
for i in range(len(flist)):
    step1 = flist[i]
    step2 = fits.open(step1)
    #align1 = step2[0].data
    data_sub = step2[0].data

    ####objects = sep.extract(data_sub, 1.5, err=bkg.globalrms)
    sep.set_extract_pixstack(5000000)
    data_sub = data_sub.byteswap().newbyteorder()
    objects = sep.extract(data_sub, 2000)

    # how many objects were detected
    len(objects)

    # plot background-subtracted image
    fig, ax = plt.subplots()
    m, s = np.mean(data_sub), np.std(data_sub)
    im = ax.imshow(data_sub,
                   interpolation='nearest',
                   cmap='gray',
                   vmin=m - s,
                   vmax=m + s,
                   origin='lower')
Ejemplo n.º 22
0
    def extract_features(self, subregions, mask=None):
        """Extract image features for each subregion. Image should be cropped
        and masked.

        :param subregions: subregions to be used
        :param mask: mask to be applied in source extraction, optional

        :return: None
        """
        # set internal pixel buffer
        sep.set_extract_pixstack(10000000)

        # extract time from header and derive frame properties
        try:
            time = Time(self.header['DATE-OBS'], format='isot')
            features = OrderedDict([
                ('time', time.isot),
                ('filename', self.filename.split(os.path.sep)[-1]),
                ('moon_alt', observatory.moon_altaz(time).alt.deg),
                ('sun_alt', observatory.sun_altaz(time).alt.deg),
                ('moon_phase', 1-observatory.moon_phase(time).value/np.pi),
            ])
        except KeyError as e:
            conf.logger.error('missing time data in file {}: {}.'.format(
                self.filename, e))
            return False

        # derive and subtract sky background
        bkg = sep.Background(self.data.astype(np.float64),
                             bw=conf.SEP_BKGBOXSIZE, bh=conf.SEP_BKGBOXSIZE,
                             fw=conf.SEP_BKGXRANGE, fh=conf.SEP_BKGYRANGE)
        data_sub = self.data - bkg.back()

        # if mask is provided, it is applied in the proper derivation of
        # source brightness thresholds
        if mask is not None:
            threshold = (np.ma.median(np.ma.array(data_sub,
                                                  mask=(1-mask))) +
                         np.median(bkg.rms())*conf.SEP_SIGMA)
            src = sep.extract(data_sub, threshold, minarea=conf.SEP_MINAREA,
                              mask=(1-mask),
                              deblend_nthresh=conf.SEP_DEBLENDN,
                              deblend_cont=conf.SEP_DEBLENDV)
        else:
            threshold = (np.median(data_sub) +
                         np.median(bkg.rms())*conf.SEP_SIGMA)
            src = sep.extract(data_sub, threshold, minarea=conf.SEP_MINAREA,
                              mask=mask,
                              deblend_nthresh=conf.SEP_DEBLENDN,
                              deblend_cont=conf.SEP_DEBLENDV)

        # apply max_flag cutoff (reject flawed sources)
        src = src[src['flag'] <= conf.SEP_MAXFLAG]

        # feature extraction per subregion
        features['srcdens'] = []
        features['bkgmedian'] = []
        features['bkgmean'] = []
        features['bkgstd'] = []
        for i, sub in enumerate(subregions):
            features['srcdens'].append(len(
                src[sub[src['y'].astype(np.int),
                        src['x'].astype(np.int)]])/np.sum(sub[mask== 1]))
            features['bkgmedian'].append(np.median(bkg.back()[sub]))
            features['bkgmean'].append(np.mean(bkg.back()[sub]))
            features['bkgstd'].append(np.std(bkg.back()[sub]))

        self.subregions = subregions
        self.features = features
Ejemplo n.º 23
0
    def do_stage(self, images):
        for i, image in enumerate(images):
            try:
                # Set the number of source pixels to be 5% of the total. This keeps us safe from
                # satellites and airplanes.
                sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))

                data = image.data.copy()
                error = (np.abs(data) + image.readnoise ** 2.0) ** 0.5
                mask = image.bpm > 0

                # Fits can be backwards byte order, so fix that if need be and subtract
                # the background
                try:
                    bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
                except ValueError:
                    data = data.byteswap(True).newbyteorder()
                    bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
                bkg.subfrom(data)

                # Do an initial source detection
                # TODO: Add back in masking after we are sure SEP works
                sources = sep.extract(data, self.threshold, minarea=self.min_area,
                                      err=error, deblend_cont=0.005)

                # Convert the detections into a table
                sources = Table(sources)

                # Calculate the ellipticity
                sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])

                # Fix any value of theta that are invalid due to floating point rounding
                # -pi / 2 < theta < pi / 2
                sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
                sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi

                # Calculate the kron radius
                kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
                                                  sources['a'], sources['b'],
                                                  sources['theta'], 6.0)
                sources['flag'] |= krflag
                sources['kronrad'] = kronrad

                # Calcuate the equivilent of flux_auto
                flux, fluxerr, flag = sep.sum_ellipse(data, sources['x'], sources['y'],
                                                      sources['a'], sources['b'],
                                                      np.pi / 2.0, 2.5 * kronrad,
                                                      subpix=1, err=error)
                sources['flux'] = flux
                sources['fluxerr'] = fluxerr
                sources['flag'] |= flag

                # Calculate the FWHMs of the stars:
                fwhm = 2.0 * (np.log(2) * (sources['a'] ** 2.0 + sources['b'] ** 2.0)) ** 0.5
                sources['fwhm'] = fwhm

                # Cut individual bright pixels. Often cosmic rays
                sources = sources[fwhm > 1.0]

                # Measure the flux profile
                flux_radii, flag = sep.flux_radius(data, sources['x'], sources['y'],
                                                   6.0 * sources['a'], [0.25, 0.5, 0.75],
                                                   normflux=sources['flux'], subpix=5)
                sources['flag'] |= flag
                sources['fluxrad25'] = flux_radii[:, 0]
                sources['fluxrad50'] = flux_radii[:, 1]
                sources['fluxrad75'] = flux_radii[:, 2]

                # Calculate the windowed positions
                sig = 2.0 / 2.35 * sources['fluxrad50']
                xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'], sig)
                sources['flag'] |= flag
                sources['xwin'] = xwin
                sources['ywin'] = ywin

                # Calculate the average background at each source
                bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(), sources['x'], sources['y'],
                                                         sources['a'], sources['b'], np.pi / 2.0,
                                                         2.5 * sources['kronrad'], subpix=1)
                #masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
                #                                         sources['a'], sources['b'], np.pi / 2.0,
                #                                         2.5 * kronrad, subpix=1)

                background_area = (2.5 * sources['kronrad']) ** 2.0 * sources['a'] * sources['b'] * np.pi # - masksum
                sources['background'] = bkgflux
                sources['background'][background_area > 0] /= background_area[background_area > 0]
                # Update the catalog to match fits convention instead of python array convention
                sources['x'] += 1.0
                sources['y'] += 1.0

                sources['xpeak'] += 1
                sources['ypeak'] += 1

                sources['xwin'] += 1.0
                sources['ywin'] += 1.0

                sources['theta'] = np.degrees(sources['theta'])

                image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
                                        'flux', 'fluxerr', 'background', 'fwhm',
                                        'a', 'b', 'theta', 'kronrad', 'ellipticity',
                                        'fluxrad25', 'fluxrad50', 'fluxrad75',
                                        'x2', 'y2', 'xy', 'flag']

                # Add the units and description to the catalogs
                image.catalog['x'].unit = 'pixel'
                image.catalog['x'].description = 'X coordinate of the object'
                image.catalog['y'].unit = 'pixel'
                image.catalog['y'].description = 'Y coordinate of the object'
                image.catalog['xwin'].unit = 'pixel'
                image.catalog['xwin'].description = 'Windowed X coordinate of the object'
                image.catalog['ywin'].unit = 'pixel'
                image.catalog['ywin'].description = 'Windowed Y coordinate of the object'
                image.catalog['xpeak'].unit = 'pixel'
                image.catalog['xpeak'].description = 'X coordinate of the peak'
                image.catalog['ypeak'].unit = 'pixel'
                image.catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
                image.catalog['flux'].unit = 'counts'
                image.catalog['flux'].description = 'Flux within a Kron-like elliptical aperture'
                image.catalog['fluxerr'].unit = 'counts'
                image.catalog['fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
                image.catalog['background'].unit = 'counts'
                image.catalog['background'].description = 'Average background value in the aperture'
                image.catalog['fwhm'].unit = 'pixel'
                image.catalog['fwhm'].description = 'FWHM of the object'
                image.catalog['a'].unit = 'pixel'
                image.catalog['a'].description = 'Semi-major axis of the object'
                image.catalog['b'].unit = 'pixel'
                image.catalog['b'].description = 'Semi-minor axis of the object'
                image.catalog['theta'].unit = 'degrees'
                image.catalog['theta'].description = 'Position angle of the object'
                image.catalog['kronrad'].unit = 'pixel'
                image.catalog['kronrad'].description = 'Kron radius used for extraction'
                image.catalog['ellipticity'].description = 'Ellipticity'
                image.catalog['fluxrad25'].unit = 'pixel'
                image.catalog['fluxrad25'].description = 'Radius containing 25% of the flux'
                image.catalog['fluxrad50'].unit = 'pixel'
                image.catalog['fluxrad50'].description = 'Radius containing 50% of the flux'
                image.catalog['fluxrad75'].unit = 'pixel'
                image.catalog['fluxrad75'].description = 'Radius containing 75% of the flux'
                image.catalog['x2'].unit = 'pixel^2'
                image.catalog['x2'].description = 'Variance on X coordinate of the object'
                image.catalog['y2'].unit = 'pixel^2'
                image.catalog['y2'].description = 'Variance on Y coordinate of the object'
                image.catalog['xy'].unit = 'pixel^2'
                image.catalog['xy'].description = 'XY covariance of the object'
                image.catalog['flag'].description = 'Bit mask combination of extraction and photometry flags'

                image.catalog.sort('flux')
                image.catalog.reverse()

                logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
                logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))

                # Save some background statistics in the header
                mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
                image.header['L1MEAN'] = (mean_background,
                                          '[counts] Sigma clipped mean of frame background')
                logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))

                median_background = np.median(bkg.back())
                image.header['L1MEDIAN'] = (median_background,
                                            '[counts] Median of frame background')
                logs.add_tag(logging_tags, 'L1MEDIAN', float(median_background))

                std_background = stats.robust_standard_deviation(bkg.back())
                image.header['L1SIGMA'] = (std_background,
                                           '[counts] Robust std dev of frame background')
                logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))

                # Save some image statistics to the header
                good_objects = image.catalog['flag'] == 0

                seeing = np.median(image.catalog['fwhm'][good_objects]) * image.pixel_scale
                image.header['L1FWHM'] = (seeing, '[arcsec] Frame FWHM in arcsec')
                logs.add_tag(logging_tags, 'L1FWHM', float(seeing))

                mean_ellipticity = stats.sigma_clipped_mean(sources['ellipticity'][good_objects],
                                                            3.0)
                image.header['L1ELLIP'] = (mean_ellipticity, 'Mean image ellipticity (1-B/A)')
                logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))

                mean_position_angle = stats.sigma_clipped_mean(sources['theta'][good_objects], 3.0)
                image.header['L1ELLIPA'] = (mean_position_angle,
                                            '[deg] PA of mean image ellipticity')
                logs.add_tag(logging_tags, 'L1ELLIPA', float(mean_position_angle))

                self.logger.info('Extracted sources', extra=logging_tags)

            except Exception as e:
                logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
                logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
                self.logger.error(e, extra=logging_tags)
        return images
Ejemplo n.º 24
0
def make_catalog(data, header):
    # Set the number of source pixels to be 5% of the total. This keeps us safe from
    # satellites and airplanes.
    sep.set_extract_pixstack(int(data.shape[1] * data.shape[0] * 0.05))

    data = data.copy()
    error = (np.abs(data) + header['RDNOISE']**2.0)**0.5
    mask = data > 0.9 * header['SATURATE']

    # Fits can be backwards byte order, so fix that if need be and subtract
    # the background
    try:
        bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
    except ValueError:
        data = data.byteswap(True).newbyteorder()
        bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
    bkg.subfrom(data)

    # Do an initial source detection
    sources = sep.extract(data,
                          THRESHOLD,
                          mask=mask,
                          minarea=MIN_AREA,
                          err=error,
                          deblend_cont=0.005)

    # Convert the detections into a table
    sources = Table(sources)

    # We remove anything with a detection flag >= 8
    # This includes memory overflows and objects that are too close the edge
    sources = sources[sources['flag'] < 8]

    sources = prune_nans_from_table(sources)

    # Calculate the ellipticity
    sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])

    # Fix any value of theta that are invalid due to floating point rounding
    # -pi / 2 < theta < pi / 2
    sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
    sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi

    # Calculate the kron radius
    kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
                                      sources['a'], sources['b'],
                                      sources['theta'], 6.0)
    sources['flag'] |= krflag
    sources['kronrad'] = kronrad

    # Calcuate the equivilent of flux_auto
    flux, fluxerr, flag = sep.sum_ellipse(data,
                                          sources['x'],
                                          sources['y'],
                                          sources['a'],
                                          sources['b'],
                                          np.pi / 2.0,
                                          2.5 * kronrad,
                                          subpix=1,
                                          err=error)
    sources['flux'] = flux
    sources['fluxerr'] = fluxerr
    sources['flag'] |= flag

    # Calculate the FWHMs of the stars:
    fwhm = 2.0 * (np.log(2) * (sources['a']**2.0 + sources['b']**2.0))**0.5
    sources['fwhm'] = fwhm

    # Cut individual bright pixels. Often cosmic rays
    sources = sources[fwhm > 1.0]

    # Update the catalog to match fits convention instead of python array convention
    sources['x'] += 1.0
    sources['y'] += 1.0

    sources['xpeak'] += 1
    sources['ypeak'] += 1

    sources['theta'] = np.degrees(sources['theta'])

    return save_catalog_meta_data(sources)