Пример #1
0
def organize_by_brick(sample_fns,
                      sbricks,
                      outdir=None,
                      seed=None,
                      prefix=None):
    '''
    For each sample_fn, split into bricks
    get  brick sample fn for that brick and sample
    write it if does not exist
    sample_fn -- sample_seed.fits file assigned to that mpi task
    sbricks -- survey bricks table cut to radec region
    '''
    dr = get_bybrick_dir(outdir=outdir)
    for sample_fn in sample_fns:
        # Skip if already looped over bricks for this sample
        check_done = os.path.join(dr, get_sample_fn(seed=seed, prefix=prefix))
        check_done = check_done.replace('.fits', '_done.txt')
        if os.path.exists(check_done):
            print('check_done exists: %s' % check_done)
            continue
        #
        sample = fits_table(sample_fn)
        print('sample min,max ra,dec= %f %f %f %f' % (sample.ra.min(),sample.ra.max(),\
                                                      sample.dec.min(),sample.dec.max()))
        # Loop over survey bricks
        survey = LegacySurveyData()
        for sbrick in sbricks:
            # Get output fn for this brick and sample
            fn = os.path.join(
                dr,
                get_brick_sample_fn(brickname=sbrick.brickname,
                                    seed=seed,
                                    prefix=prefix))
            if os.path.exists(fn):
                continue
            # Cut sample by brick's bounds
            brickinfo = survey.get_brick_by_name(sbrick.brickname)
            brickwcs = wcs_for_brick(brickinfo)
            ra1, ra2, dec1, dec2 = brickwcs.radec_bounds()
            keep=  (sample.ra >= ra1)*(sample.ra <= ra2)*\
                   (sample.dec >= dec1)*(sample.dec <= dec2)
            sample2 = sample.copy()
            if np.where(keep)[0].size > 0:
                sample2.cut(keep)
                sample2.writeto(fn)
                print('Wrote %s' % fn)
            else:
                print('WARNING: sample=%s has no ra,dec in brick=%s' %
                      (sample_fn, sbrick.brickname))
        # This sample is done
        with open(check_done, 'w') as foo:
            foo.write('done')
Пример #2
0
def getbrickfiles(brickname=None):

    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    ccdinfo = survey.ccds_touching_wcs(brickwcs)
    nccd = len(ccdinfo)

    calibdir = survey.get_calib_dir()
    imagedir = survey.survey_dir

    # Construct image file names and the calibration file names.
    expnum = ccdinfo.expnum
    ccdname = ccdinfo.ccdname

    psffiles = list()
    skyfiles = list()
    imagefiles = list()
    for ccd in ccdinfo:
        info = survey.get_image_object(ccd)
        for attr in ['imgfn', 'dqfn', 'wtfn']:
            fn = getattr(info, attr).replace(imagedir+'/', '')
            #if '160108_073601' in fn:
            #    pdb.set_trace()
            imagefiles.append(fn)
        psffiles.append(info.psffn.replace(calibdir, 'calib'))
        skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))
        
    #for ii in range(nccd):
        #exp = '{0:08d}'.format(expnum[ii])
        #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
        #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
        #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
        #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))

    #print(np.array(imagefiles))
    #print(np.array(psffiles))
    #print(np.array(skyfiles))
    return imagefiles, psffiles, skyfiles
Пример #3
0
def getbrickfiles(brickname=None):

    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    ccdinfo = survey.ccds_touching_wcs(brickwcs)
    nccd = len(ccdinfo)

    calibdir = survey.get_calib_dir()
    imagedir = survey.survey_dir

    # Construct image file names and the calibration file names.
    expnum = ccdinfo.expnum
    ccdname = ccdinfo.ccdname

    psffiles = list()
    skyfiles = list()
    imagefiles = list()
    for ccd in ccdinfo:
        info = survey.get_image_object(ccd)
        for attr in ['imgfn', 'dqfn', 'wtfn']:
            fn = getattr(info, attr).replace(imagedir + '/', '')
            #if '160108_073601' in fn:
            #    pdb.set_trace()
            imagefiles.append(fn)
        psffiles.append(info.psffn.replace(calibdir, 'calib'))
        skyfiles.append(info.splineskyfn.replace(calibdir, 'calib'))

    #for ii in range(nccd):
    #exp = '{0:08d}'.format(expnum[ii])
    #rootfile = os.path.join(exp[:5], exp, 'decam-'+exp+'-'+ccdname[ii]+'.fits')
    #psffiles.append(os.path.join('calib', 'decam', 'psfex', rootfile))
    #skyfiles.append(os.path.join('calib', 'decam', 'splinesky', rootfile))
    #imagefiles.append(os.path.join('images', str(np.core.defchararray.strip(ccdinfo.image_filename[ii]))))

    #print(np.array(imagefiles))
    #print(np.array(psffiles))
    #print(np.array(skyfiles))
    return imagefiles, psffiles, skyfiles
Пример #4
0
            if task is None:
                break
            try:
                print('Task:', task.task)

                brickname = task.task
                print('Checking for existing out file')
                # shortcut
                dirnm = os.path.join('depthcuts', brickname[:3])
                outfn = os.path.join(dirnm, 'ccds-%s.fits' % brickname)
                if os.path.exists(outfn):
                    print('Exists:', outfn)
                    task.set_state(qdo.Task.SUCCEEDED)
                    continue
                print('Getting brick', brickname)
                brick = survey.get_brick_by_name(brickname)
                print('Got brick, running depth cut')
                rtn = run_one_brick((brick, 0, 1, False, kwargs))
                if rtn != 0:
                    allgood = rtn
                print('Done, result', rtn)
                if rtn == 0:
                    task.set_state(qdo.Task.SUCCEEDED)
                else:
                    task.set_state(qdo.Task.FAILED, err=1)
            except:
                import traceback
                traceback.print_exc()
                task.set_state(qdo.Task.FAILED, err=1)
        sys.exit(0)
Пример #5
0
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--plots', action='store_true')
    parser.add_argument('--brick', help='Brick name to run')
    parser.add_argument(
        '--input-dir',
        default='/global/projecta/projectdirs/cosmo/work/legacysurvey/dr7')
    #/global/cscratch1/sd/desiproc/dr7out')
    parser.add_argument('--survey-dir',
                        default='/global/cscratch1/sd/dstn/dr7-depthcut')
    parser.add_argument('--output-dir',
                        default='/global/cscratch1/sd/dstn/bright')
    opt = parser.parse_args()

    plots = opt.plots
    ps = PlotSequence('bright')
    brickname = opt.brick

    insurvey = LegacySurveyData(opt.input_dir, cache_dir=opt.survey_dir)
    outsurvey = LegacySurveyData(opt.output_dir, output_dir=opt.output_dir)

    bfn = insurvey.find_file('blobmap', brick=brickname)
    print('Found blob map', bfn)
    blobs = fitsio.read(bfn)
    h, w = blobs.shape

    brick = insurvey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brick)
    radius = np.sqrt(2.) * 0.25 * 1.01
    neighbors = insurvey.get_bricks_near(brick.ra, brick.dec, radius)
    print(len(neighbors), 'bricks nearby')

    def showbool(X):
        d = downsample_max(X, 8)
        h, w = X.shape
        plt.imshow(d,
                   interpolation='nearest',
                   origin='lower',
                   vmin=0,
                   vmax=1,
                   extent=[0, w, 0, h],
                   cmap='gray')

    brightblobs = set()

    for nb in neighbors:
        if nb.brickname == brickname:
            # ignore myself!
            continue
        print('Neighbor:', nb.brickname)

        mfn = insurvey.find_file('maskbits', brick=nb.brickname)
        if not os.path.exists(mfn):
            print('No maskbits file:', mfn)
            continue
        maskbits = fitsio.read(mfn)
        bright = ((maskbits & MASKBITS['BRIGHT']) > 0)
        print(np.sum(bright > 0), 'BRIGHT pixels set')
        primary = (maskbits & MASKBITS['NPRIMARY'] == 0)
        print(np.sum(primary), 'PRIMARY pixels set')
        edge = binary_dilation(primary, structure=np.ones((3, 3), bool))
        edge = edge * np.logical_not(primary)
        brightedge = edge & bright

        if plots:
            plt.clf()
            showbool(bright)
            plt.title('bright: brick %s' % nb.brickname)
            ps.savefig()

            # plt.clf()
            # showbool(primary)
            # plt.title('PRIMARY, brick %s' % nb.brickname)
            # ps.savefig()
            #
            # plt.clf()
            # showbool(edge)
            # plt.title('boundary, brick %s' % nb.brickname)
            # ps.savefig()

            plt.clf()
            showbool(brightedge)
            plt.title('bright at edge, brick %s' % nb.brickname)
            ps.savefig()

        nwcs = wcs_for_brick(nb)

        yy, xx = np.nonzero(brightedge)
        print(len(yy), 'bright edge pixels')
        if len(yy) == 0:
            continue
        rr, dd = nwcs.pixelxy2radec(xx + 1, yy + 1)
        print('RA range', rr.min(), rr.max(), 'vs brick', brick.ra1, brick.ra2)
        print('Dec range', dd.min(), dd.max(), 'vs brick', brick.dec1,
              brick.dec2)
        # Find pixels that are within this brick's unique area
        I, = np.nonzero((rr >= brick.ra1) * (rr <= brick.ra2) *
                        (dd >= brick.dec1) * (dd <= brick.dec2))

        if plots:
            plt.clf()
            plt.plot(
                [brick.ra1, brick.ra1, brick.ra2, brick.ra2, brick.ra1],
                [brick.dec1, brick.dec2, brick.dec2, brick.dec1, brick.dec1],
                'b-')
            plt.plot(rr, dd, 'k.')
            plt.plot(rr[I], dd[I], 'r.')
            plt.title('Bright pixels from %s' % nb.brickname)
            ps.savefig()

        if len(I) == 0:
            print('No edge pixels touch')
            #plt.plot(br,bd, 'b-')
            continue
        #print('Edge pixels touch!')
        #plt.plot(br,bd, 'r-', zorder=20)

        ok, x, y = brickwcs.radec2pixelxy(rr[I], dd[I])
        x = np.round(x).astype(int) - 1
        y = np.round(y).astype(int) - 1
        print('Pixel ranges X', x.min(), x.max(), 'Y', y.min(), y.max())
        assert (np.all((x >= 0) * (x < w) * (y >= 0) * (y < h)))
        print('Adding blobs:', np.unique(blobs[y, x]))
        brightblobs.update(blobs[y, x])
        print('Blobs touching bright pixels:', brightblobs)

    print()
    brightblobs.discard(-1)
    if len(brightblobs) == 0:
        print('No neighboring bright blobs to update!')
        return
    print('Updating', len(brightblobs), 'blobs:', brightblobs)

    tmap = np.zeros(blobs.max() + 2, bool)
    for b in brightblobs:
        tmap[b + 1] = True
    touching = tmap[blobs + 1]

    if plots:
        plt.clf()
        showbool(touching)
        plt.title('Blobs touching bright, brick %s' % brickname)
        ps.savefig()

    mfn = insurvey.find_file('maskbits', brick=brickname)
    maskbits, hdr = fitsio.read(mfn, header=True)
    updated = maskbits | (MASKBITS['BRIGHT'] * touching)
    if np.all(maskbits == updated):
        print('No bits updated!  (Bright stars were already masked)')
        return
    maskbits = updated

    if plots:
        plt.clf()
        showbool((maskbits & MASKBITS['BRIGHT']) > 0)
        plt.title('New maskbits map for BRIGHT, brick %s' % brickname)
        ps.savefig()

    with outsurvey.write_output('maskbits', brick=brickname) as out:
        out.fits.write(maskbits, hdr=hdr)

    tfn = insurvey.find_file('tractor', brick=brickname)
    phdr = fitsio.read_header(tfn, ext=0)
    hdr = fitsio.read_header(tfn, ext=1)
    T = fits_table(tfn)
    print('Read', len(T), 'sources')
    print('Bright:', Counter(T.brightstarinblob))
    iby = np.clip(np.round(T.by), 0, h - 1).astype(int)
    ibx = np.clip(np.round(T.bx), 0, w - 1).astype(int)
    if plots:
        before = np.flatnonzero(T.brightstarinblob)
    T.brightstarinblob |= touching[iby, ibx]
    print('Bright:', Counter(T.brightstarinblob))

    # yuck -- copy the TUNIT headers from input to output.
    units = [
        hdr.get('TUNIT%i' % (i + 1), '') for i in range(len(T.get_columns()))
    ]

    if plots:
        plt.clf()
        showbool((maskbits & MASKBITS['BRIGHT']) > 0)
        ax = plt.axis()
        after = np.flatnonzero(T.brightstarinblob)
        plt.plot(T.bx[before], T.by[before], 'gx')
        plt.plot(T.bx[after], T.by[after], 'r.')
        plt.axis(ax)
        plt.title('sources with brightstarinblob, brick %s' % brickname)
        ps.savefig()

    with outsurvey.write_output('tractor', brick=brickname) as out:
        T.writeto(None,
                  fits_object=out.fits,
                  primheader=phdr,
                  header=hdr,
                  units=units)
Пример #6
0
class SimStamps(object):
    """Object for exracting sim cutouts

        Args:
            ls_dir: LEGACY_SURVEY_DIR, like 'tests/end_to_end/testcase_DR5_grz'
            outdir: path to dir containing obiwan,coadd,tractor dirs
    """
    def __init__(self, ls_dir=None, outdir=None, savedir=None, jpeg=False):
        """outdir: required
           ls_dir: not needed if env var LEGACY_SURVEY_DIR already set
           save_dir: where write hdf5 files, outdir if None
        """
        self.outdir = outdir
        self.jpeg = jpeg
        if ls_dir:
            os.environ["LEGACY_SURVEY_DIR"] = ls_dir
        self.savedir = savedir
        if self.savedir is None:
            self.savedir = self.outdir
        self.survey = LegacySurveyData()

    def get_brickwcs(self, brick):
        brickinfo = self.survey.get_brick_by_name(brick)
        self.brickwcs = wcs_for_brick(brickinfo)

    def load_data(self, brick, cat_fn, coadd_dir):
        """loads coadd and catalogue data

        Args:
            brick:
            coadd_dir: path/to/rs0, rs300, rs300_skipid, etc
        """
        print('Loading ra,dec from %s' % (os.path.dirname))
        self.cat = fits_table(cat_fn)

        print('Loading from %s' % coadd_dir)
        self.img_fits, self.ivar_fits = {}, {}
        for b in self.bands:
            self.img_fits[b] = readImage(
                os.path.join(coadd_dir,
                             'legacysurvey-%s-image-%s.fits.fz' % (brick, b)))
            self.ivar_fits[b] = readImage(
                os.path.join(coadd_dir,
                             'legacysurvey-%s-invvar-%s.fits.fz' % (brick, b)))
        self.img_jpeg = readImage(os.path.join(
            coadd_dir, 'legacysurvey-%s-image.jpg' % (brick)),
                                  jpeg=True)
        # galsim.Image() so can determine overlap w/cutouts
        self.img_gs, self.ivar_gs, self.jpeg_gs = {}, {}, {}
        for ib, b in enumerate(self.bands):
            self.img_gs[b] = galsim.Image(self.img_fits[b])
            self.ivar_gs[b] = galsim.Image(self.ivar_fits[b])

    def extract(self, hw=32):
        """For each id,x,y in self.cat, extracts image cutout

        Args:
            hw: half-width, pixels, (hw*2) x (hw*2) image cutout
        """
        for cat in self.cat:
            xslc = slice(cat.x - hw, cat.x + hw)
            yslc = slice(cat.y - hw, cat.y + hw)
            # N x N x Number of bands
            test_img = galsim.Image(np.zeros((2 * hw + 1, 2 * hw + 1)))
            # y,x because numpy indexing
            test_img.setCenter(cat.x, cat.y)
            olap = test_img.bounds & self.img_gs[self.bands[0]].bounds
            assert (olap.area() > 0)
            if olap.numpyShape() == test_img.array.shape:
                # Grab from fits image b/c aligned better
                _ = self.hdf5_obj.create_dataset(str(cat.id)+'/img',
                                                 chunks=True, \
                    data= np.array([sliceImage(self.img_fits[band],
                                               xslice=xslc,yslice=yslc)
                                    for band in self.bands_str]).T)
                _ = self.hdf5_obj.create_dataset(str(cat.id)+'/ivar',
                                                 chunks=True, \
                    data= np.array([sliceImage(self.ivar_fits[band],
                                               xslice=xslc,yslice=yslc)
                                    for band in self.bands_str]).T)

                _ = self.hdf5_jpeg.create_dataset(str(cat.id)+'/img',
                                                  chunks=True,dtype=np.uint8, \
                    data= sliceImage(self.img_jpeg,
                                     xslice=xslc,yslice=yslc))
                # _ = self.hdf5_obj.create_dataset(str(cat.id)+'/img',
                #                                  chunks=True, \
                #     data= np.array([self.img_fits[band][olap].array
                #                     for band in self.bands_str]).T)
                # _ = self.hdf5_obj.create_dataset(str(cat.id)+'/ivar',
                #                                  chunks=True, \
                #     data= np.array([self.ivar_fits[band][olap].array
                #                     for band in self.bands_str]).T)

            else:
                # On edge
                # Note, galsim.Image() cannot be 3D
                img = [test_img.copy()] * len(self.bands)
                ivar = [test_img.copy()] * len(self.bands)
                for i, band in enumerate(self.bands_str):
                    img[i][olap] += self.img_gs[band][olap]
                    ivar[i][olap] += self.ivar_gs[band][olap]

                _ = self.hdf5_obj_onedge.create_dataset(str(cat.id)+'/img',
                                                        chunks=True, \
                    data= np.array([d.array for d in img]).T)
                _ = self.hdf5_obj_onedge.create_dataset(str(cat.id)+'/ivar',
                                                        chunks=True, \
                    data= np.array([d.array for d in ivar]).T)

    def run(self, brick, stampSize=64, applyCuts=True, zoom=None):
        """Write the hdf5 image files for all rs/* in this brick

        Args:
            brick: brickname
            stampSize: height and width in pixes of training image
            zoom: if legacypipe was run with zoom option
        """
        self.get_brickwcs(brick)
        # coadd fits images must exist
        self.set_paths_to_data(brick)
        coadd_fns = glob(os.path.join(self.coadd_dirs[0], '*-image-*.fits.fz'))
        if len(coadd_fns) == 0:
            raise IOError('no image.fits.fz file here: %s' %
                          self.coadd_dirs[0])
        # set of bands in this brick
        self.bands = (pd.Series(coadd_fns).str.replace('.fits.fz',
                                                       '').str[-1].values)
        assert (self.bands.size > 0)
        # One hdf5 file for this brick, like '_rz.hdf5'
        self.bands_str = ''.join(sorted(self.bands))
        assert (self.bands_str in HDF5_KEYS)
        self.set_output_fns(brick, self.bands_str)

        if os.path.exists(self.hdf5_fn):
            try:
                f = h5py.File(self.hdf5_fn, 'r')
                f2 = h5py.File(self.hdf5_fn_onedge, 'r')
                f3 = h5py.File(self.hdf5_fn_jpeg, 'r')
                if (len(f.keys()) == 0) & (len(f2.keys()) == 0):
                    # remove empty files then make them
                    for fn in [
                            self.hdf5_fn, self.hdf5_fn_onedge,
                            self.hdf5_fn_jpeg
                    ]:
                        dobash('rm %s' % fn)
                else:
                    # processing done, skip this brick
                    print('Skipping %s, hdf5 already filled: %s' %
                          (brick, self.hdf5_fn))
                    return None
            except OSError:
                # One of these got messed up, redo it
                for fn in [
                        self.hdf5_fn, self.hdf5_fn_onedge, self.hdf5_fn_jpeg
                ]:
                    os.remove(fn)
                    print('removed ', fn)
        self.hdf5_obj = h5py.File(self.hdf5_fn, "w")
        self.hdf5_obj_onedge = h5py.File(self.hdf5_fn_onedge, "w")
        self.hdf5_jpeg = h5py.File(self.hdf5_fn_jpeg, "w")
        # Many rs*/ dirs per brick
        for cat_fn, coadd_dir in zip(self.cat_fns, self.coadd_dirs):
            if (self.jpeg) & (os.path.basename(coadd_dir) != 'rs0'):
                print('jpeg=True, so skipping %s' % coadd_dir)
                continue
            self.load_data(brick, cat_fn, coadd_dir)
            self.set_xyid(zoom=zoom)
            if applyCuts:
                self.apply_cuts()
                self.write_mag_sorted_ids()
            self.extract(hw=int(stampSize / 2))
        self.hdf5_obj.close()
        self.hdf5_obj_onedge.close()
        self.hdf5_jpeg.close()
        print('Wrote %s' % self.hdf5_fn)
        print('Wrote %s' % self.hdf5_fn_onedge)
        print('Wrote %s' % self.hdf5_fn_jpeg)

    def set_paths_to_data(self, brick):
        """lists of catalogues filenames and coadd dirs"""
        search = os.path.join(self.outdir, 'coadd', brick[:3], brick, '*rs*',
                              'legacysurvey-%s-ccds.fits' % brick)
        rs_dirs = glob(search)
        rs_dirs = [os.path.basename(os.path.dirname(a)) for a in rs_dirs]
        if len(rs_dirs) == 0:
            raise IOError('No rs dirs here: %s' % search)
        self.cat_fns,self.coadd_dirs= \
            zip(*[(os.path.join(self.outdir,'obiwan',
                                brick[:3],brick,rs_dir,
                                'simcat-elg-%s.fits' % brick),
                   os.path.join(self.outdir,'coadd',
                                brick[:3],brick,rs_dir)
                   )
                   for rs_dir in rs_dirs])

    def set_output_fns(self, brick, bands_str):
        dr = os.path.join(self.savedir, 'hdf5', brick[:3], brick)
        # hdf5
        self.hdf5_fn = os.path.join(dr, 'img_ivar_%s.hdf5' % bands_str)
        self.hdf5_fn_onedge = self.hdf5_fn.replace('.hdf5', '_onedge.hdf5')
        self.hdf5_fn_jpeg = self.hdf5_fn.replace('img_ivar_', 'jpeg_')
        # table of mag sorted ids
        self.sorted_ids_fn = self.hdf5_fn.replace(
            os.path.basename(self.hdf5_fn), 'sorted_ids.fits')
        try:
            dobash('mkdir -p %s' % dr)
        except ValueError:
            print('hdf5 dir already exists: ', dr)

    def set_xyid(self, zoom=None):
        _, x, y = self.brickwcs.radec2pixelxy(self.cat.ra, self.cat.dec)
        if zoom:
            x -= zoom[0]
            y -= zoom[2]
        self.cat.set('x', x.astype(int))
        self.cat.set('y', y.astype(int))
        assert ('id' in self.cat.get_columns())

    def apply_cuts(self):
        len_bef = len(self.cat)
        print('After cut, have %d/%d' % (len(self.cat), len_bef))

    def write_mag_sorted_ids(self, band='g'):
        mag = self.get_mag(band)
        inds = np.argsort(
            mag)  # small to large (mag, so brightest to faintest)
        T = fits_table()
        T.set('id', self.cat.id)
        T.set('mag_' + band, mag)
        T = T[inds]
        T.writeto(self.sorted_ids_fn)
        print('Wrote %s' % self.sorted_ids_fn)

    def get_mag(self, band='g'):
        return flux2mag(self.cat.get(band + 'flux'))
Пример #7
0
def main():
    import argparse
    parser = argparse.ArgumentParser()

    parser.add_argument(
        '--dr',
        '--drdir',
        dest='drdir',
        default='/project/projectdirs/cosmo/data/legacysurvey/dr5',
        help='Directory containing data release w/ tar-gzipped calibs')
    parser.add_argument(
        '-b',
        '--brick',
        help='Brick name to run; required unless --radec is given')
    parser.add_argument(
        '--radec',
        nargs=2,
        help='RA,Dec center for a custom location (not a brick)')
    parser.add_argument('--pixscale',
                        type=float,
                        default=0.262,
                        help='Pixel scale of the output coadds (arcsec/pixel)')
    parser.add_argument('-W',
                        '--width',
                        type=int,
                        default=3600,
                        help='Target image width, default %(default)i')
    parser.add_argument('-H',
                        '--height',
                        type=int,
                        default=3600,
                        help='Target image height, default %(default)i')
    parser.add_argument(
        '--zoom',
        type=int,
        nargs=4,
        help='Set target image extent (default "0 3600 0 3600")')
    parser.add_argument('--no-psf',
                        dest='do_psf',
                        default=True,
                        action='store_false',
                        help='Do not extract PsfEx files')
    parser.add_argument('--no-sky',
                        dest='do_sky',
                        default=True,
                        action='store_false',
                        help='Do not extract SplineSky files')

    opt = parser.parse_args()
    if opt.brick is None and opt.radec is None:
        parser.print_help()
        return -1
    optdict = vars(opt)

    drdir = opt.drdir
    W = opt.width
    H = opt.height
    pixscale = opt.pixscale
    target_extent = opt.zoom

    do_psf = opt.do_psf
    do_sky = opt.do_sky

    #brickname = '1501p020'

    custom = (opt.radec is not None)
    #ra,dec = 216.03, 34.86
    if custom:
        ra, dec = opt.radec  #27.30, -10.43
        ra = float(ra)
        dec = float(dec)
        #W,H = 1000,1000
        #W,H = 1500,1500
        brickname = 'custom_%.3f_%.3f' % (ra, dec)
        #do_sky = False

    survey = LegacySurveyData()

    if custom:
        from legacypipe.survey import BrickDuck
        # Custom brick; create a fake 'brick' object
        brick = BrickDuck(ra, dec, brickname)
    else:
        brickname = opt.brick
        brick = survey.get_brick_by_name(brickname)

    # Get WCS object describing brick
    targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
    if target_extent is not None:
        (x0, x1, y0, y1) = target_extent
        W = x1 - x0
        H = y1 - y0
        targetwcs = targetwcs.get_subimage(x0, y0, W, H)

    # Find CCDs
    ccds = survey.ccds_touching_wcs(targetwcs, ccdrad=None)
    if ccds is None:
        raise NothingToDoError('No CCDs touching brick')
    print(len(ccds), 'CCDs touching target WCS')

    for ccd in ccds:
        im = survey.get_image_object(ccd)
        print('CCD', im)

        expnum = '%08i' % im.expnum

        if do_psf:
            if os.path.exists(im.psffn) or os.path.exists(im.merged_psffn):
                print('PSF file exists')
            else:
                print('Need PSF', im.psffn, im.merged_psffn)

                tarfn = os.path.join(
                    drdir, 'calib', im.camera, 'psfex-merged',
                    'legacysurvey_dr5_calib_decam_psfex-merged_%s.tar.gz' %
                    expnum[:5])
                print(tarfn)
                if os.path.exists(tarfn):
                    outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
                    cmd = 'cd %s/%s/psfex-merged && tar xvzf %s %s' % (
                        survey.get_calib_dir(), im.camera, tarfn, outfn)
                    print(cmd)
                    os.system(cmd)

        if do_sky:
            if os.path.exists(im.splineskyfn) or os.path.exists(
                    im.merged_splineskyfn):
                print('Sky file exists')
            else:
                print('Need sky', im.splineskyfn, im.merged_splineskyfn)

                tarfn = os.path.join(
                    drdir, 'calib', im.camera, 'splinesky-merged',
                    'legacysurvey_dr5_calib_decam_splinesky-merged_%s.tar.gz' %
                    expnum[:5])
                print(tarfn)
                if os.path.exists(tarfn):
                    outfn = '%s/%s-%s.fits' % (expnum[:5], im.camera, expnum)
                    cmd = 'cd %s/%s/splinesky-merged && tar xvzf %s %s' % (
                        survey.get_calib_dir(), im.camera, tarfn, outfn)
                    print(cmd)
                    os.system(cmd)
Пример #8
0
def main(args=None):
    """Main routine which parses the optional inputs."""
    # Command line options
    parser= get_parser()    
    args = parser.parse_args(args=args)
    # Setup loggers
    if args.verbose:
        lvl = logging.DEBUG
    else:
        lvl = logging.INFO
    logging.basicConfig(level=lvl, stream=sys.stdout) #,format='%(message)s')
    log = logging.getLogger('decals_sim')
    # Sort through args 
    log.info('decals_sim.py args={}'.format(args))
    max_nobj=500
    max_nchunk=1000
    if args.ith_chunk is not None: assert(args.ith_chunk <= max_nchunk-1)
    assert(args.nchunk <= max_nchunk)
    assert(args.nobj <= max_nobj)
    if args.ith_chunk is not None: 
        assert(args.nchunk == 1) #if choose a chunk, only doing 1 chunk
    if args.nobj is None:
        parser.print_help()
        sys.exit(1)
 
    brickname = args.brick
    objtype = args.objtype.upper()
    lobjtype = objtype.lower()

    for obj in ('LRG', 'LSB', 'QSO'):
        if objtype == obj:
            log.warning('{} objtype not yet supported!'.format(objtype))
            return 0

    # Deal with the paths.
    if 'DECALS_SIM_DIR' in os.environ:
        decals_sim_dir = os.getenv('DECALS_SIM_DIR')
    else:
        decals_sim_dir = '.'
        
    nobj = args.nobj
    nchunk = args.nchunk
    rand = np.random.RandomState(args.seed) # determines seed for all chunks
    seeds = rand.random_integers(0,2**18, max_nchunk)

    log.info('Object type = {}'.format(objtype))
    log.info('Number of objects = {}'.format(nobj))
    log.info('Number of chunks = {}'.format(nchunk))

    # Optionally zoom into a portion of the brick
    survey = LegacySurveyData()
    brickinfo = survey.get_brick_by_name(brickname)
    brickwcs = wcs_for_brick(brickinfo)
    W, H, pixscale = brickwcs.get_width(), brickwcs.get_height(), brickwcs.pixel_scale()

    log.info('Brick = {}'.format(brickname))
    if args.zoom is not None: # See also runbrick.stage_tims()
        (x0, x1, y0, y1) = args.zoom
        W = x1 - x0
        H = y1 - y0
        brickwcs = brickwcs.get_subimage(x0, y0, W, H)
        log.info('Zoom (pixel boundaries) = {}'.format(args.zoom))
    targetrd = np.array([brickwcs.pixelxy2radec(x, y) for x, y in
                         [(1,1), (W,1), (W,H), (1,H), (1,1)]])
 
    radec_center = brickwcs.radec_center()
    log.info('RA, Dec center = {}'.format(radec_center))
    log.info('Brick = {}'.format(brickname))
    
    if args.ith_chunk is not None: 
        chunk_list= [args.ith_chunk]
    else: 
        chunk_list= range(nchunk)

    # Store args in dict for easy func passing
    kwargs=dict(seeds=seeds,\
                brickname=brickname, \
                decals_sim_dir= decals_sim_dir,\
                brickwcs= brickwcs, \
                objtype=objtype,\
                lobjtype=lobjtype,\
                nobj=nobj,\
                nchunk=nchunk,\
                args=args) 
    
    # Create simulated catalogues and run Tractor
    create_metadata(kwargs=kwargs)
    # do chunks
    for ith_chunk in chunk_list:
        log.info('Working on chunk {:02d}/{:02d}'.format(ith_chunk,kwargs['nchunk']-1))
        # Random ra,dec and source properties
        create_ith_simcat(ith_chunk, d=kwargs)
        # Run tractor
        do_one_chunk(d=kwargs)
        # Clean up output
        do_ith_cleanup(ith_chunk=ith_chunk, d=kwargs)
    log.info('All done!')
Пример #9
0
from legacypipe.survey import LegacySurveyData, wcs_for_brick, SimpleGalaxy
from tractor import NanoMaggies
import numpy as np
import pylab as plt

survey = LegacySurveyData()

brick = survey.get_brick_by_name('0006m062')
print('Brick', brick)
targetwcs = wcs_for_brick(brick)
H, W = targetwcs.shape
targetrd = np.array([
    targetwcs.pixelxy2radec(x, y)
    for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
print('Target Ra,Dec:', targetrd)

ccds = survey.ccds_touching_wcs(targetwcs)

#ccds = survey.find_ccds(expnum=239662, ccdname='S18')
ccds = ccds[(ccds.expnum == 239662) * (ccds.ccdname == 'S18')]
assert (len(ccds) == 1)
ccd = ccds[0]
im = survey.get_image_object(ccd)
tim = im.get_tractor_image(pixPsf=True,
                           hybridPsf=True,
                           splinesky=True,
                           radecpoly=targetrd)

fullpsf = tim.psf
th, tw = tim.shape