Example #1
0
 def run_psfex(self, surveyname):
     from astrometry.util.file import trymakedirs
     from legacypipe.survey import get_git_version
     sedir = self.survey.get_se_dir()
     trymakedirs(self.psffn, dir=True)
     primhdr = self.read_image_primary_header()
     plver = primhdr.get('PLVER', '')
     verstr = get_git_version()
     cmds = [
         'psfex -c %s -PSF_DIR %s %s' %
         (os.path.join(sedir, surveyname + '.psfex'),
          os.path.dirname(self.psffn), self.sefn),
         'modhead %s LEGPIPEV %s "legacypipe git version"' %
         (self.psffn, verstr),
         'modhead %s PLVER %s "CP ver of image file"' % (self.psffn, plver)
     ]
     for cmd in cmds:
         print(cmd)
         rtn = os.system(cmd)
         if rtn:
             raise RuntimeError('Command failed: %s: return value: %i' %
                                (cmd, rtn))
Example #2
0
def run_one_brick(X):
    brick, ibrick, nbricks, plots, kwargs = X

    survey = LegacySurveyData()

    print()
    print()
    print('Brick', (ibrick + 1), 'of', nbricks, ':', brick.brickname)

    dirnm = os.path.join('depthcuts', brick.brickname[:3])
    outfn = os.path.join(dirnm, 'ccds-%s.fits' % brick.brickname)
    if os.path.exists(outfn):
        print('Exists:', outfn)
        return 0

    H, W = 3600, 3600
    pixscale = 0.262
    bands = ['g', 'r', 'z']

    # Get WCS object describing brick
    targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
    targetrd = np.array([
        targetwcs.pixelxy2radec(x, y)
        for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
    ])
    gitver = get_git_version()

    ccds = survey.ccds_touching_wcs(targetwcs)
    if ccds is None:
        print('No CCDs actually touching brick')
        return 0
    print(len(ccds), 'CCDs actually touching brick')

    ccds.cut(np.in1d(ccds.filter, bands))
    print('Cut on filter:', len(ccds), 'CCDs remain.')

    if 'ccd_cuts' in ccds.get_columns():
        norig = len(ccds)
        ccds.cut(ccds.ccd_cuts == 0)
        print(len(ccds), 'of', norig, 'CCDs pass cuts')
    else:
        print('No CCD cuts')

    if len(ccds) == 0:
        print('No CCDs left')
        return 0

    ps = None
    if plots:
        from astrometry.util.plotutils import PlotSequence
        ps = PlotSequence('depth-%s' % brick.brickname)

    splinesky = True
    gaussPsf = False
    pixPsf = True
    do_calibs = False
    normalizePsf = True

    get_depth_maps = kwargs.pop('get_depth_maps', False)

    try:
        D = make_depth_cut(survey,
                           ccds,
                           bands,
                           targetrd,
                           brick,
                           W,
                           H,
                           pixscale,
                           plots,
                           ps,
                           splinesky,
                           gaussPsf,
                           pixPsf,
                           normalizePsf,
                           do_calibs,
                           gitver,
                           targetwcs,
                           get_depth_maps=get_depth_maps,
                           **kwargs)
        if get_depth_maps:
            keep, overlapping, depthmaps = D
        else:
            keep, overlapping = D
    except:
        print('Failed to make_depth_cut():')
        import traceback
        traceback.print_exc()
        return -1

    print(np.sum(overlapping), 'CCDs overlap the brick')
    print(np.sum(keep), 'CCDs passed depth cut')
    ccds.overlapping = overlapping
    ccds.passed_depth_cut = keep

    if not os.path.exists(dirnm):
        try:
            os.makedirs(dirnm)
        except:
            pass

    if get_depth_maps:
        for band, depthmap in depthmaps:
            doutfn = os.path.join(dirnm,
                                  'depth-%s-%s.fits' % (brick.brickname, band))
            hdr = fitsio.FITSHDR()
            # Plug the WCS header cards into these images
            targetwcs.add_to_header(hdr)
            hdr.delete('IMAGEW')
            hdr.delete('IMAGEH')
            hdr.add_record(dict(name='EQUINOX', value=2000.))
            hdr.add_record(dict(name='FILTER', value=band))
            fitsio.write(doutfn, depthmap, header=hdr)
            print('Wrote', doutfn)

    tmpfn = os.path.join(os.path.dirname(outfn),
                         'tmp-' + os.path.basename(outfn))
    ccds.writeto(tmpfn)
    os.rename(tmpfn, outfn)
    print('Wrote', outfn)

    return 0
Example #3
0
def main(args=None):
    import os
    import datetime
    import logging
    import numpy as np
    from legacypipe.survey import get_git_version
    from legacypipe.runbrick import (get_parser, get_runbrick_kwargs,
                                     run_brick, NothingToDoError,
                                     RunbrickError)

    print()
    print('mpi-runbrick.py starting at', datetime.datetime.now().isoformat())
    print('legacypipe git version:', get_git_version())
    if args is None:
        print('Command-line args:', sys.argv)
        cmd = 'python'
        for vv in sys.argv:
            cmd += ' {}'.format(vv)
        print(cmd)
    else:
        print('Args:', args)
    print()

    parser = get_parser()
    opt = parser.parse_args(args=args)

    if opt.brick is None and opt.radec is None:
        parser.print_help()
        return -1

    optdict = vars(opt)
    verbose = optdict.pop('verbose')

    survey, kwargs = get_runbrick_kwargs(**optdict)
    if kwargs in [-1, 0]:
        return kwargs
    kwargs.update(command_line=' '.join(sys.argv))

    if verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    global_init(lvl)

    # matplotlib config directory, when running in shifter...
    for tag in ['CACHE', 'CONFIG']:
        if not 'XDG_%s_HOME' % tag in os.environ:
            src = os.path.expanduser('~/.%s' % (tag.lower()))
            # Read-only?
            if os.path.isdir(src) and not (os.access(src, os.W_OK)):
                import shutil
                import tempfile
                tempdir = tempfile.mkdtemp(prefix='%s-' % tag.lower())
                os.rmdir(tempdir)
                shutil.copytree(src, tempdir,
                                symlinks=True)  #dirs_exist_ok=True)
                # astropy config file: if XDG_CONFIG_HOME is set,
                # expect to find $XDG_CONFIG_HOME/astropy, not
                # ~/.astropy.
                if tag == 'CONFIG':
                    shutil.copytree(os.path.expanduser('~/.astropy'),
                                    os.path.join(tempdir, 'astropy'),
                                    symlinks=True)  #dirs_exist_ok=True)
                os.environ['XDG_%s_HOME' % tag] = tempdir

    if opt.plots:
        import matplotlib
        matplotlib.use('Agg')
        import pylab as plt
        plt.figure(figsize=(12, 9))
        plt.subplots_adjust(left=0.07,
                            right=0.99,
                            bottom=0.07,
                            top=0.93,
                            hspace=0.2,
                            wspace=0.05)

    # The "initializer" arg is only available in mpi4py master
    pool = MyMPIPool(initializer=global_init, initargs=(lvl, ))

    u = int(os.environ.get('OMPI_UNIVERSE_SIZE', '0'))
    if u == 0:
        u = int(os.environ.get('MPICH_UNIVERSE_SIZE', '0'))
    if u == 0:
        from mpi4py import MPI
        u = MPI.COMM_WORLD.Get_size()

    print('Booting up MPI pool with', u, 'workers...')
    pool.bootup()
    print('Booted up MPI pool.')
    pool._processes = u
    kwargs.update(pool=pool)

    #pool.map(hello, np.arange(128))

    rtn = -1
    try:
        run_brick(opt.brick, survey, **kwargs)
        rtn = 0
    except NothingToDoError as e:
        print()
        if hasattr(e, 'message'):
            print(e.message)
        else:
            print(e)
        print()
        rtn = 0
    except RunbrickError as e:
        print()
        if hasattr(e, 'message'):
            print(e.message)
        else:
            print(e)
        print()
        rtn = -1

    print('Shutting down MPI pool...')
    pool.shutdown()
    print('Shut down MPI pool')

    return rtn
Example #4
0
# Setup.py, you suck
#setup_keywords['test_suite'] = 'nose.collector'
#setup_keywords['tests_require'] = ['nose']
#setup_keywords['test_suite'] = 'py/test'
#setup_keywords['test_suite'] = 'run_tests'
#
# Import this module to get __doc__ and __version__.
#
sys.path.insert(int(sys.path[0] == ''), './py')
try:
    from importlib import import_module
    product = import_module(setup_keywords['name'])
    setup_keywords['long_description'] = product.__doc__

    from legacypipe.survey import get_git_version
    version = get_git_version(os.path.dirname(__file__)).replace('-', '.')

    setup_keywords['version'] = version
except ImportError:
    #
    # Try to get the long description from the README.rst file.
    #
    if os.path.exists('README.rst'):
        with open('README.rst') as readme:
            setup_keywords['long_description'] = readme.read()
    else:
        setup_keywords['long_description'] = ''
    setup_keywords['version'] = 'dr3.dev'

#
# Set other keywords for the setup function.  These are automated, & should
Example #5
0
# setup_keywords['tests_require'] = ['nose']
# setup_keywords['test_suite'] = 'py/test'
# setup_keywords['test_suite'] = 'run_tests'
#
# Import this module to get __doc__ and __version__.
#
sys.path.insert(int(sys.path[0] == ""), "./py")
try:
    from importlib import import_module

    product = import_module(setup_keywords["name"])
    setup_keywords["long_description"] = product.__doc__

    from legacypipe.survey import get_git_version

    version = get_git_version(os.path.dirname(__file__)).replace("-", ".")

    setup_keywords["version"] = version
except ImportError:
    #
    # Try to get the long description from the README.rst file.
    #
    if os.path.exists("README.rst"):
        with open("README.rst") as readme:
            setup_keywords["long_description"] = readme.read()
    else:
        setup_keywords["long_description"] = ""
    setup_keywords["version"] = "dr3.dev"

#
# Set other keywords for the setup function.  These are automated, & should
Example #6
0
def main():
    sbfn = 'skybricks.fits'
    SB = fits_table(sbfn)

    Bnorth = fits_table(
        '/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/north/survey-bricks-dr9-north.fits.gz'
    )
    Bsouth = fits_table(
        '/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/survey-bricks-dr9-south.fits.gz'
    )
    Bnorth.cut(Bnorth.survey_primary)
    Bsouth.cut(Bsouth.survey_primary)
    #Bsouth.cut(Bsouth.dec > -30)
    Bnorth.hemi = np.array(['north'] * len(Bnorth))
    Bsouth.hemi = np.array(['south'] * len(Bsouth))
    B = merge_tables([Bnorth, Bsouth])

    # Rough cut the skybricks to those near bricks.
    I, J, d = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., nearest=True)
    SB.cut(I)

    import argparse
    parser = argparse.ArgumentParser()
    #parser.add_argument('--brick', help='Sky brick name')
    parser.add_argument('--minra',
                        type=float,
                        help='Cut to a minimum RA range of sky bricks')
    parser.add_argument('--maxra',
                        type=float,
                        help='Cut to a maximum RA range of sky bricks')
    parser.add_argument('--threads',
                        type=int,
                        help='Parallelize on this many cores')
    opt = parser.parse_args()
    if opt.minra:
        SB.cut(SB.ra >= opt.minra)
    if opt.maxra:
        SB.cut(SB.ra <= opt.maxra)
    version = get_git_version(os.getcwd())
    print('Version string:', version)

    # Find bricks near skybricks, as a rough cut.
    # (magic 1. degree > hypot(skybrick radius, brick radiu) ~= 0.9)
    Inear = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., indexlist=True)

    args = []
    k = 0
    Isb = []
    for isb, (sb, inear) in enumerate(zip(SB, Inear)):
        if inear is None:
            continue
        args.append((k, sb, B[np.array(inear)], version))
        k += 1
        Isb.append(isb)
    print(len(args), 'sky bricks')
    SB.cut(np.array(Isb))

    if opt.threads:
        mp = multiproc(opt.threads)
    else:
        mp = multiproc()

    exist = mp.map(run_one, args)

    if opt.minra is None and opt.maxra is None:
        exist = np.array(exist)
        SB[exist].writeto('skybricks-exist.fits')
    return
Example #7
0
    return


if __name__ == '__main__':
    #main()
    #B = fits_table('bricks-near.fits')
    #SB = fits_table('skybricks.fits')
    B = fits_table(
        '/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/north/survey-bricks-dr9-north.fits.gz'
    )
    B.cut(B.dec > 77)
    B.hemi = np.array(['north'] * len(B))

    SB = fits_table(
        '/global/cfs/cdirs/desi/target/skybricks/v3/skybricks-exist.fits')
    #I = np.flatnonzero(np.isin(SB.brickname, [
    #    '2187p340', '2175p340',
    #'1500p290', '1507p300', '1519p300',
    #'1505p310', '1490p320'
    #]))
    I = np.flatnonzero(SB.dec >= 78.)

    version = get_git_version(os.getcwd())
    #version = 4

    mp = multiproc(32)
    args = []
    for i in I:
        args.append((0, SB[i], B, version))
    mp.map(run_one, args)