Esempio n. 1
0
def main():
    from legacypipe.runbrick import run_brick, get_parser, get_runbrick_kwargs
    parser = get_parser()
    #parser.add_argument('--subset', type=int, help='COSMOS subset number [0 to 4, 10 to 12]', default=0)
    opt = parser.parse_args()
    if opt.brick is None and opt.radec is None:
        parser.print_help()
        return -1

    optdict = vars(opt)
    verbose = optdict.pop('verbose')

    survey, kwargs = get_runbrick_kwargs(**optdict)
    if kwargs in [-1,0]:
        return kwargs

    #kwargs.update(prereqs_update={'detect': 'mask_junk',
    #                              'galdetect': 'mask_junk'})

    kwargs.update(prereqs_update={'detect': 'tims'})
    
    stagefunc = CallGlobalTime('stage_%s', globals())
    kwargs.update(stagefunc=stagefunc)

    #kwargs.update(stages=['image_coadds', 'detect'])
    #kwargs.update(stages=['galdetect'])
    #kwargs.update(stages=['detect',]) # 'srcs']) # with early_coadds, srcs:image_coadds
    #kwargs.update(stages=['srcs'])

    run_brick(opt.brick, survey, **kwargs)
    return 0
Esempio n. 2
0
def get_runbrick_setup(**kwargs):
    """Convert runbrick.py cmd line options into `**kwargs` for run_brick()

    The command line options depend on the Data Release (e.g. the
        legacypipe code version. The cmd line options associated with
        each DR get modified and repackaged into a dict in
        legacypipe.runbrick so this converter is required to call run_brick
        appropriately

    Args:
        **kwargs: dict of the cmd line options to obiwan.kenobi.py

    Returns:
        dict to use when calling legacypipe.runbrick.run_brick like
            run_brick(brickname, survey, `**dict`)
    """
    dataset = kwargs['dataset']
    assert (dataset in DATASETS)
    from legacypipe.runbrick import get_runbrick_kwargs
    from legacypipe.runbrick import get_parser as get_runbrick_parser
    zm = kwargs['zoom']
    cmd_line = [
        '--no-write', '--skip', '--force-all', '--zoom',
        '%d' % zm[0],
        '%d' % zm[1],
        '%d' % zm[2],
        '%d' % zm[3], '--no-wise', '--threads',
        '%d' % kwargs['threads']
    ]
    if kwargs['checkpoint']:
        checkpoint_fn = get_checkpoint_fn(kwargs['outdir'], kwargs['brick'],
                                          kwargs['rowstart'])
        cmd_line += ['--checkpoint', checkpoint_fn]
    if kwargs['stage']:
        cmd_line += ['--stage', kwargs['stage']]
    if kwargs['early_coadds']:
        cmd_line += ['--early-coadds', '--stage', 'image_coadds']
    if kwargs['skip_ccd_cuts']:
        cmd_line += ['--skip_ccd_cuts']
    #if kwargs['stage']:
    #    cmd_line += ['--stage', '%s' % kwargs['stage']]
    if dataset == 'dr3':
        #cmd_line += ['--hybrid-psf']
        cmd_line += ['--run', 'dr3', '--nsigma', '6', '--simp']
    elif dataset == 'dr5':
        # defaults: rex (use --simp), nsigma 6 ,hybrid-psf (--no-hybrid-psf otherwise)
        # depth cut already done (use --depth-cut to do depth cut anyway)
        cmd_line += ['--run', 'dr5']

    rb_parser = get_runbrick_parser()
    rb_opt = rb_parser.parse_args(args=cmd_line)
    rb_optdict = vars(rb_opt)
    # remove keys as Dustin' does
    _ = rb_optdict.pop('ps', None)
    _ = rb_optdict.pop('verbose', None)
    _, rb_kwargs = get_runbrick_kwargs(**rb_optdict)
    return rb_kwargs
Esempio n. 3
0
def main():

    from astrometry.util.fits import fits_table, merge_tables
    from legacypipe.survey import exposure_metadata
    # Fake up a survey-ccds.fits table from MzLS_CP
    from glob import glob
    #fns = glob('/project/projectdirs/cosmo/staging/mosaicz/MZLS_CP/CP20160202/k4m_160203_*oki*')
    fns = glob('/project/projectdirs/cosmo/staging/mosaicz/MZLS_CP/CP20160202/k4m_160203_08*oki*')

    print('Filenames:', fns)
    T = exposure_metadata(fns)

    # HACK
    T.fwhm = T.seeing / 0.262

    # FAKE
    T.ccdnmatch = np.zeros(len(T), np.int32) + 50
    T.zpt = np.zeros(len(T), np.float32) + 26.518
    T.ccdzpt = T.zpt.copy()
    T.ccdraoff = np.zeros(len(T), np.float32)
    T.ccddecoff = np.zeros(len(T), np.float32)
    
    fmap = {'zd':'z'}
    T.filter = np.array([fmap[f] for f in T.filter])
    
    T.writeto('mzls-ccds.fits')

    os.system('cp mzls-ccds.fits ~/legacypipe-dir/survey-ccds.fits')
    os.system('gzip -f ~/legacypipe-dir/survey-ccds.fits')
    
    import sys
    sys.exit(0)
    
    import logging
    import sys
    from legacypipe.runbrick import run_brick, get_runbrick_kwargs, get_parser

    parser = get_parser()
    opt = parser.parse_args()
    if opt.brick is None and opt.radec is None:
        parser.print_help()
        return -1
    survey, kwargs = get_runbrick_kwargs(opt)
    if kwargs in [-1, 0]:
        return kwargs

    if opt.verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)

    kwargs.update(splinesky=True, pixPsf=True)

    run_brick(opt.brick, survey, **kwargs)
Esempio n. 4
0
def main():
    import logging
    import sys
    from legacypipe.runbrick import run_brick, get_runbrick_kwargs, get_parser

    parser = get_parser()
    opt = parser.parse_args()
    if opt.brick is None and opt.radec is None:
        parser.print_help()
        return -1
    kwargs = get_runbrick_kwargs(opt)
    if kwargs in [-1, 0]:
        return kwargs

    if opt.verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)

    kwargs.update(splinesky=True, pixPsf=True)

    run_brick(opt.brick, **kwargs)
Esempio n. 5
0
def main():
    import logging
    import sys
    from legacypipe.runbrick import run_brick, get_runbrick_kwargs, get_parser

    parser = get_parser()
    opt = parser.parse_args()
    if opt.brick is None and opt.radec is None:
        parser.print_help()
        return -1
    kwargs = get_runbrick_kwargs(opt)
    if kwargs in [-1, 0]:
        return kwargs

    if opt.verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)

    kwargs.update(splinesky=True, pixPsf=True)

    run_brick(opt.brick, **kwargs)
Esempio n. 6
0
def main(args=None):
    import os
    import datetime
    import logging
    import numpy as np
    from legacypipe.survey import get_git_version
    from legacypipe.runbrick import (get_parser, get_runbrick_kwargs,
                                     run_brick, NothingToDoError,
                                     RunbrickError)

    print()
    print('mpi-runbrick.py starting at', datetime.datetime.now().isoformat())
    print('legacypipe git version:', get_git_version())
    if args is None:
        print('Command-line args:', sys.argv)
        cmd = 'python'
        for vv in sys.argv:
            cmd += ' {}'.format(vv)
        print(cmd)
    else:
        print('Args:', args)
    print()

    parser = get_parser()
    opt = parser.parse_args(args=args)

    if opt.brick is None and opt.radec is None:
        parser.print_help()
        return -1

    optdict = vars(opt)
    verbose = optdict.pop('verbose')

    survey, kwargs = get_runbrick_kwargs(**optdict)
    if kwargs in [-1, 0]:
        return kwargs
    kwargs.update(command_line=' '.join(sys.argv))

    if verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    global_init(lvl)

    # matplotlib config directory, when running in shifter...
    for tag in ['CACHE', 'CONFIG']:
        if not 'XDG_%s_HOME' % tag in os.environ:
            src = os.path.expanduser('~/.%s' % (tag.lower()))
            # Read-only?
            if os.path.isdir(src) and not (os.access(src, os.W_OK)):
                import shutil
                import tempfile
                tempdir = tempfile.mkdtemp(prefix='%s-' % tag.lower())
                os.rmdir(tempdir)
                shutil.copytree(src, tempdir,
                                symlinks=True)  #dirs_exist_ok=True)
                # astropy config file: if XDG_CONFIG_HOME is set,
                # expect to find $XDG_CONFIG_HOME/astropy, not
                # ~/.astropy.
                if tag == 'CONFIG':
                    shutil.copytree(os.path.expanduser('~/.astropy'),
                                    os.path.join(tempdir, 'astropy'),
                                    symlinks=True)  #dirs_exist_ok=True)
                os.environ['XDG_%s_HOME' % tag] = tempdir

    if opt.plots:
        import matplotlib
        matplotlib.use('Agg')
        import pylab as plt
        plt.figure(figsize=(12, 9))
        plt.subplots_adjust(left=0.07,
                            right=0.99,
                            bottom=0.07,
                            top=0.93,
                            hspace=0.2,
                            wspace=0.05)

    # The "initializer" arg is only available in mpi4py master
    pool = MyMPIPool(initializer=global_init, initargs=(lvl, ))

    u = int(os.environ.get('OMPI_UNIVERSE_SIZE', '0'))
    if u == 0:
        u = int(os.environ.get('MPICH_UNIVERSE_SIZE', '0'))
    if u == 0:
        from mpi4py import MPI
        u = MPI.COMM_WORLD.Get_size()

    print('Booting up MPI pool with', u, 'workers...')
    pool.bootup()
    print('Booted up MPI pool.')
    pool._processes = u
    kwargs.update(pool=pool)

    #pool.map(hello, np.arange(128))

    rtn = -1
    try:
        run_brick(opt.brick, survey, **kwargs)
        rtn = 0
    except NothingToDoError as e:
        print()
        if hasattr(e, 'message'):
            print(e.message)
        else:
            print(e)
        print()
        rtn = 0
    except RunbrickError as e:
        print()
        if hasattr(e, 'message'):
            print(e.message)
        else:
            print(e)
        print()
        rtn = -1

    print('Shutting down MPI pool...')
    pool.shutdown()
    print('Shut down MPI pool')

    return rtn
Esempio n. 7
0
def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--old-output',
        required=True,
        help='"Old" output directory to read old checksum file from.')

    parser.add_argument('-b',
                        '--brick',
                        required=True,
                        help='Brick name to run')
    parser.add_argument('-P',
                        '--pickle',
                        dest='pickle_pat',
                        help='Pickle filename pattern, default %(default)s',
                        default='pickles/runbrick-%(brick)s-%%(stage)s.pickle')
    parser.add_argument('-n',
                        '--no-write',
                        dest='write',
                        default=True,
                        action='store_false')
    parser.add_argument(
        '--survey-dir',
        type=str,
        default=None,
        help='Override the $LEGACY_SURVEY_DIR environment variable')
    parser.add_argument('-d',
                        '--outdir',
                        dest='output_dir',
                        help='Set output base directory, default "."')

    opt = parser.parse_args()
    optdict = vars(opt)

    old_output_dir = optdict.pop('old_output')

    from legacypipe.runbrick import get_runbrick_kwargs
    survey, kwargs = get_runbrick_kwargs(**optdict)
    if kwargs in [-1, 0]:
        return kwargs

    import logging
    lvl = logging.INFO
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
    # tractor logging is *soooo* chatty
    logging.getLogger('tractor.engine').setLevel(lvl + 10)

    from legacypipe.survey import LegacySurveyData
    old_survey = LegacySurveyData(survey_dir=old_output_dir,
                                  output_dir=old_output_dir)
    kwargs.update(old_survey=old_survey)
    brickname = optdict['brick']

    from astrometry.util.stages import CallGlobalTime, runstage

    prereqs = {
        'outliers': None,
    }
    prereqs.update({'merge_checksums': 'outliers'})

    pickle_pat = optdict['pickle_pat']
    pickle_pat = pickle_pat % dict(brick=brickname)

    stagefunc = CallGlobalTime('stage_%s', globals())
    stage = 'merge_checksums'
    R = runstage(stage,
                 pickle_pat,
                 stagefunc,
                 prereqs=prereqs,
                 force=[stage],
                 write=[],
                 **kwargs)
Esempio n. 8
0
def main():
    from astrometry.util.ttime import Time

    t0 = Time()

    parser = get_parser()
    parser.set_defaults(wise=False)

    parser.add_argument('expnum', type=int, help='Exposure number')
    parser.add_argument('ccdname', help='CCD name (eg: "N4")')

    opt = parser.parse_args()
    optdict = vars(opt)
    verbose = optdict.pop('verbose')

    import logging
    import sys
    if verbose == 0:
        lvl = logging.INFO
    else:
        lvl = logging.DEBUG
    logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
    # tractor logging is *soooo* chatty
    logging.getLogger('tractor.engine').setLevel(lvl + 10)

    expnum = optdict.pop('expnum')
    ccdname = optdict.pop('ccdname')
    survey = FakeLegacySurveyData(survey_dir=opt.survey_dir,
                                  output_dir=opt.output_dir,
                                  cache_dir=opt.cache_dir)
    survey.no_kd = False

    ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname)
    if len(ccds) == 0:
        print('Did not find EXPNUM', expnum, 'CCDNAME', ccdname)
        return -1

    # Force the CCDs
    survey.ccds = ccds
    survey.no_kd = True
    ccd = ccds[0]
    print('Found CCD', ccd)

    awcs = survey.get_approx_wcs(ccd)
    ra,dec = awcs.radec_center()
    h,w = awcs.shape
    rr,dd = awcs.pixelxy2radec([1,1,w,w], [1,h,h,1])
    # Rotate RAs to be around RA=180 to avoid wrap-around
    rotra = np.fmod((rr - ra + 180) + 360, 360.)

    # assume default pixscale
    pixscale = 0.262 / 3600

    W = int(np.ceil((rotra.max() - rotra.min()) * np.cos(np.deg2rad(dec))
                    / pixscale))
    H = int(np.ceil((dd.max() - dd.min()) / pixscale))
    print('W, H', W, H)

    optdict.update(survey=survey)
    survey, kwargs = get_runbrick_kwargs(**optdict)

    # Only set W,H if they were not specified (to other than default values) on the command-line
    if opt.width == 3600 and opt.height == 3600:
        kwargs.update(width=W, height=H)
    if opt.radec is None and opt.brick is None:
        kwargs.update(radec=(ra,dec))
    kwargs.update(bands=[ccd.filter])
    print('kwargs:', kwargs)
    run_brick(None, survey, **kwargs)
    print('Finished:', Time()-t0)
Esempio n. 9
0
def main():
    from astrometry.util.ttime import Time

    t0 = Time()

    parser = get_parser()
    parser.set_defaults(wise=False)

    #hybridPsf=True, normalizePsf=True, rex=True, splinesky=True,
    #gaia_stars=True, wise=False, ceres=False,

    parser.add_argument('expnum', type=int, help='Exposure number')
    parser.add_argument('ccdname', help='CCD name (eg: "N4")')

    opt = parser.parse_args()
    optdict = vars(opt)
    verbose = optdict.pop('verbose')

    expnum = optdict.pop('expnum')
    ccdname = optdict.pop('ccdname')

    #print('optdict:', optdict)

    survey = FakeLegacySurveyData(survey_dir=opt.survey_dir,
                                  output_dir=opt.output_dir,
                                  cache_dir=opt.cache_dir)
    survey.no_kd = False

    ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname)
    if len(ccds) == 0:
        print('Did not find EXPNUM', expnum, 'CCDNAME', ccdname)
        return -1

    # Force the CCDs
    survey.ccds = ccds
    survey.no_kd = True

    ccd = ccds[0]
    print('Found CCD', ccd)

    awcs = survey.get_approx_wcs(ccd)
    ra, dec = awcs.radec_center()
    h, w = awcs.shape
    rr, dd = awcs.pixelxy2radec([1, 1, w, w], [1, h, h, 1])
    # Rotate RAs to be around RA=180 to avoid wrap-around
    rotra = np.fmod((rr - ra + 180) + 360, 360.)

    # assume default pixscale
    pixscale = 0.262 / 3600

    W = int(
        np.ceil(
            (rotra.max() - rotra.min()) * np.cos(np.deg2rad(dec)) / pixscale))
    H = int(np.ceil((dd.max() - dd.min()) / pixscale))
    print('W, H', W, H)

    optdict.update(survey=survey)

    survey, kwargs = get_runbrick_kwargs(**optdict)

    kwargs.update(radec=(ra, dec), width=W, height=H, bands=[ccd.filter])

    #if opt.brick is None and opt.radec is None:

    run_brick(None, survey, **kwargs)

    #hybridPsf=True, normalizePsf=True, rex=True, splinesky=True,
    #gaia_stars=True, wise=False, ceres=False,

    print('Finished:', Time() - t0)