def main():
from legacypipe.runbrick import run_brick, get_parser, get_runbrick_kwargs
parser = get_parser()
#parser.add_argument('--subset', type=int, help='COSMOS subset number [0 to 4, 10 to 12]', default=0)
opt = parser.parse_args()
if opt.brick is None and opt.radec is None:
parser.print_help()
return -1
optdict = vars(opt)
verbose = optdict.pop('verbose')
survey, kwargs = get_runbrick_kwargs(**optdict)
if kwargs in [-1,0]:
return kwargs
#kwargs.update(prereqs_update={'detect': 'mask_junk',
# 'galdetect': 'mask_junk'})
kwargs.update(prereqs_update={'detect': 'tims'})
stagefunc = CallGlobalTime('stage_%s', globals())
kwargs.update(stagefunc=stagefunc)
#kwargs.update(stages=['image_coadds', 'detect'])
#kwargs.update(stages=['galdetect'])
#kwargs.update(stages=['detect',]) # 'srcs']) # with early_coadds, srcs:image_coadds
#kwargs.update(stages=['srcs'])
run_brick(opt.brick, survey, **kwargs)
return 0
def get_runbrick_setup(**kwargs):
"""Convert runbrick.py cmd line options into `**kwargs` for run_brick()
The command line options depend on the Data Release (e.g. the
legacypipe code version. The cmd line options associated with
each DR get modified and repackaged into a dict in
legacypipe.runbrick so this converter is required to call run_brick
appropriately
Args:
**kwargs: dict of the cmd line options to obiwan.kenobi.py
Returns:
dict to use when calling legacypipe.runbrick.run_brick like
run_brick(brickname, survey, `**dict`)
"""
dataset = kwargs['dataset']
assert (dataset in DATASETS)
from legacypipe.runbrick import get_runbrick_kwargs
from legacypipe.runbrick import get_parser as get_runbrick_parser
zm = kwargs['zoom']
cmd_line = [
'--no-write', '--skip', '--force-all', '--zoom',
'%d' % zm[0],
'%d' % zm[1],
'%d' % zm[2],
'%d' % zm[3], '--no-wise', '--threads',
'%d' % kwargs['threads']
]
if kwargs['checkpoint']:
checkpoint_fn = get_checkpoint_fn(kwargs['outdir'], kwargs['brick'],
kwargs['rowstart'])
cmd_line += ['--checkpoint', checkpoint_fn]
if kwargs['stage']:
cmd_line += ['--stage', kwargs['stage']]
if kwargs['early_coadds']:
cmd_line += ['--early-coadds', '--stage', 'image_coadds']
if kwargs['skip_ccd_cuts']:
cmd_line += ['--skip_ccd_cuts']
#if kwargs['stage']:
# cmd_line += ['--stage', '%s' % kwargs['stage']]
if dataset == 'dr3':
#cmd_line += ['--hybrid-psf']
cmd_line += ['--run', 'dr3', '--nsigma', '6', '--simp']
elif dataset == 'dr5':
# defaults: rex (use --simp), nsigma 6 ,hybrid-psf (--no-hybrid-psf otherwise)
# depth cut already done (use --depth-cut to do depth cut anyway)
cmd_line += ['--run', 'dr5']
rb_parser = get_runbrick_parser()
rb_opt = rb_parser.parse_args(args=cmd_line)
rb_optdict = vars(rb_opt)
# remove keys as Dustin' does
_ = rb_optdict.pop('ps', None)
_ = rb_optdict.pop('verbose', None)
_, rb_kwargs = get_runbrick_kwargs(**rb_optdict)
return rb_kwargs
def main():
from astrometry.util.fits import fits_table, merge_tables
from legacypipe.survey import exposure_metadata
# Fake up a survey-ccds.fits table from MzLS_CP
from glob import glob
#fns = glob('/project/projectdirs/cosmo/staging/mosaicz/MZLS_CP/CP20160202/k4m_160203_*oki*')
fns = glob('/project/projectdirs/cosmo/staging/mosaicz/MZLS_CP/CP20160202/k4m_160203_08*oki*')
print('Filenames:', fns)
T = exposure_metadata(fns)
# HACK
T.fwhm = T.seeing / 0.262
# FAKE
T.ccdnmatch = np.zeros(len(T), np.int32) + 50
T.zpt = np.zeros(len(T), np.float32) + 26.518
T.ccdzpt = T.zpt.copy()
T.ccdraoff = np.zeros(len(T), np.float32)
T.ccddecoff = np.zeros(len(T), np.float32)
fmap = {'zd':'z'}
T.filter = np.array([fmap[f] for f in T.filter])
T.writeto('mzls-ccds.fits')
os.system('cp mzls-ccds.fits ~/legacypipe-dir/survey-ccds.fits')
os.system('gzip -f ~/legacypipe-dir/survey-ccds.fits')
import sys
sys.exit(0)
import logging
import sys
from legacypipe.runbrick import run_brick, get_runbrick_kwargs, get_parser
parser = get_parser()
opt = parser.parse_args()
if opt.brick is None and opt.radec is None:
parser.print_help()
return -1
survey, kwargs = get_runbrick_kwargs(opt)
if kwargs in [-1, 0]:
return kwargs
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
kwargs.update(splinesky=True, pixPsf=True)
run_brick(opt.brick, survey, **kwargs)
def main():
import logging
import sys
from legacypipe.runbrick import run_brick, get_runbrick_kwargs, get_parser
parser = get_parser()
opt = parser.parse_args()
if opt.brick is None and opt.radec is None:
parser.print_help()
return -1
kwargs = get_runbrick_kwargs(opt)
if kwargs in [-1, 0]:
return kwargs
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
kwargs.update(splinesky=True, pixPsf=True)
run_brick(opt.brick, **kwargs)
def main():
import logging
import sys
from legacypipe.runbrick import run_brick, get_runbrick_kwargs, get_parser
parser = get_parser()
opt = parser.parse_args()
if opt.brick is None and opt.radec is None:
parser.print_help()
return -1
kwargs = get_runbrick_kwargs(opt)
if kwargs in [-1, 0]:
return kwargs
if opt.verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
kwargs.update(splinesky=True, pixPsf=True)
run_brick(opt.brick, **kwargs)
def main(args=None):
import os
import datetime
import logging
import numpy as np
from legacypipe.survey import get_git_version
from legacypipe.runbrick import (get_parser, get_runbrick_kwargs,
run_brick, NothingToDoError,
RunbrickError)
print()
print('mpi-runbrick.py starting at', datetime.datetime.now().isoformat())
print('legacypipe git version:', get_git_version())
if args is None:
print('Command-line args:', sys.argv)
cmd = 'python'
for vv in sys.argv:
cmd += ' {}'.format(vv)
print(cmd)
else:
print('Args:', args)
print()
parser = get_parser()
opt = parser.parse_args(args=args)
if opt.brick is None and opt.radec is None:
parser.print_help()
return -1
optdict = vars(opt)
verbose = optdict.pop('verbose')
survey, kwargs = get_runbrick_kwargs(**optdict)
if kwargs in [-1, 0]:
return kwargs
kwargs.update(command_line=' '.join(sys.argv))
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
global_init(lvl)
# matplotlib config directory, when running in shifter...
for tag in ['CACHE', 'CONFIG']:
if not 'XDG_%s_HOME' % tag in os.environ:
src = os.path.expanduser('~/.%s' % (tag.lower()))
# Read-only?
if os.path.isdir(src) and not (os.access(src, os.W_OK)):
import shutil
import tempfile
tempdir = tempfile.mkdtemp(prefix='%s-' % tag.lower())
os.rmdir(tempdir)
shutil.copytree(src, tempdir,
symlinks=True) #dirs_exist_ok=True)
# astropy config file: if XDG_CONFIG_HOME is set,
# expect to find $XDG_CONFIG_HOME/astropy, not
# ~/.astropy.
if tag == 'CONFIG':
shutil.copytree(os.path.expanduser('~/.astropy'),
os.path.join(tempdir, 'astropy'),
symlinks=True) #dirs_exist_ok=True)
os.environ['XDG_%s_HOME' % tag] = tempdir
if opt.plots:
import matplotlib
matplotlib.use('Agg')
import pylab as plt
plt.figure(figsize=(12, 9))
plt.subplots_adjust(left=0.07,
right=0.99,
bottom=0.07,
top=0.93,
hspace=0.2,
wspace=0.05)
# The "initializer" arg is only available in mpi4py master
pool = MyMPIPool(initializer=global_init, initargs=(lvl, ))
u = int(os.environ.get('OMPI_UNIVERSE_SIZE', '0'))
if u == 0:
u = int(os.environ.get('MPICH_UNIVERSE_SIZE', '0'))
if u == 0:
from mpi4py import MPI
u = MPI.COMM_WORLD.Get_size()
print('Booting up MPI pool with', u, 'workers...')
pool.bootup()
print('Booted up MPI pool.')
pool._processes = u
kwargs.update(pool=pool)
#pool.map(hello, np.arange(128))
rtn = -1
try:
run_brick(opt.brick, survey, **kwargs)
rtn = 0
except NothingToDoError as e:
print()
if hasattr(e, 'message'):
print(e.message)
else:
print(e)
print()
rtn = 0
except RunbrickError as e:
print()
if hasattr(e, 'message'):
print(e.message)
else:
print(e)
print()
rtn = -1
print('Shutting down MPI pool...')
pool.shutdown()
print('Shut down MPI pool')
return rtn
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'--old-output',
required=True,
help='"Old" output directory to read old checksum file from.')
parser.add_argument('-b',
'--brick',
required=True,
help='Brick name to run')
parser.add_argument('-P',
'--pickle',
dest='pickle_pat',
help='Pickle filename pattern, default %(default)s',
default='pickles/runbrick-%(brick)s-%%(stage)s.pickle')
parser.add_argument('-n',
'--no-write',
dest='write',
default=True,
action='store_false')
parser.add_argument(
'--survey-dir',
type=str,
default=None,
help='Override the $LEGACY_SURVEY_DIR environment variable')
parser.add_argument('-d',
'--outdir',
dest='output_dir',
help='Set output base directory, default "."')
opt = parser.parse_args()
optdict = vars(opt)
old_output_dir = optdict.pop('old_output')
from legacypipe.runbrick import get_runbrick_kwargs
survey, kwargs = get_runbrick_kwargs(**optdict)
if kwargs in [-1, 0]:
return kwargs
import logging
lvl = logging.INFO
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
from legacypipe.survey import LegacySurveyData
old_survey = LegacySurveyData(survey_dir=old_output_dir,
output_dir=old_output_dir)
kwargs.update(old_survey=old_survey)
brickname = optdict['brick']
from astrometry.util.stages import CallGlobalTime, runstage
prereqs = {
'outliers': None,
}
prereqs.update({'merge_checksums': 'outliers'})
pickle_pat = optdict['pickle_pat']
pickle_pat = pickle_pat % dict(brick=brickname)
stagefunc = CallGlobalTime('stage_%s', globals())
stage = 'merge_checksums'
R = runstage(stage,
pickle_pat,
stagefunc,
prereqs=prereqs,
force=[stage],
write=[],
**kwargs)
def main():
from astrometry.util.ttime import Time
t0 = Time()
parser = get_parser()
parser.set_defaults(wise=False)
parser.add_argument('expnum', type=int, help='Exposure number')
parser.add_argument('ccdname', help='CCD name (eg: "N4")')
opt = parser.parse_args()
optdict = vars(opt)
verbose = optdict.pop('verbose')
import logging
import sys
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
# tractor logging is *soooo* chatty
logging.getLogger('tractor.engine').setLevel(lvl + 10)
expnum = optdict.pop('expnum')
ccdname = optdict.pop('ccdname')
survey = FakeLegacySurveyData(survey_dir=opt.survey_dir,
output_dir=opt.output_dir,
cache_dir=opt.cache_dir)
survey.no_kd = False
ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname)
if len(ccds) == 0:
print('Did not find EXPNUM', expnum, 'CCDNAME', ccdname)
return -1
# Force the CCDs
survey.ccds = ccds
survey.no_kd = True
ccd = ccds[0]
print('Found CCD', ccd)
awcs = survey.get_approx_wcs(ccd)
ra,dec = awcs.radec_center()
h,w = awcs.shape
rr,dd = awcs.pixelxy2radec([1,1,w,w], [1,h,h,1])
# Rotate RAs to be around RA=180 to avoid wrap-around
rotra = np.fmod((rr - ra + 180) + 360, 360.)
# assume default pixscale
pixscale = 0.262 / 3600
W = int(np.ceil((rotra.max() - rotra.min()) * np.cos(np.deg2rad(dec))
/ pixscale))
H = int(np.ceil((dd.max() - dd.min()) / pixscale))
print('W, H', W, H)
optdict.update(survey=survey)
survey, kwargs = get_runbrick_kwargs(**optdict)
# Only set W,H if they were not specified (to other than default values) on the command-line
if opt.width == 3600 and opt.height == 3600:
kwargs.update(width=W, height=H)
if opt.radec is None and opt.brick is None:
kwargs.update(radec=(ra,dec))
kwargs.update(bands=[ccd.filter])
print('kwargs:', kwargs)
run_brick(None, survey, **kwargs)
print('Finished:', Time()-t0)
def main():
from astrometry.util.ttime import Time
t0 = Time()
parser = get_parser()
parser.set_defaults(wise=False)
#hybridPsf=True, normalizePsf=True, rex=True, splinesky=True,
#gaia_stars=True, wise=False, ceres=False,
parser.add_argument('expnum', type=int, help='Exposure number')
parser.add_argument('ccdname', help='CCD name (eg: "N4")')
opt = parser.parse_args()
optdict = vars(opt)
verbose = optdict.pop('verbose')
expnum = optdict.pop('expnum')
ccdname = optdict.pop('ccdname')
#print('optdict:', optdict)
survey = FakeLegacySurveyData(survey_dir=opt.survey_dir,
output_dir=opt.output_dir,
cache_dir=opt.cache_dir)
survey.no_kd = False
ccds = survey.find_ccds(expnum=expnum, ccdname=ccdname)
if len(ccds) == 0:
print('Did not find EXPNUM', expnum, 'CCDNAME', ccdname)
return -1
# Force the CCDs
survey.ccds = ccds
survey.no_kd = True
ccd = ccds[0]
print('Found CCD', ccd)
awcs = survey.get_approx_wcs(ccd)
ra, dec = awcs.radec_center()
h, w = awcs.shape
rr, dd = awcs.pixelxy2radec([1, 1, w, w], [1, h, h, 1])
# Rotate RAs to be around RA=180 to avoid wrap-around
rotra = np.fmod((rr - ra + 180) + 360, 360.)
# assume default pixscale
pixscale = 0.262 / 3600
W = int(
np.ceil(
(rotra.max() - rotra.min()) * np.cos(np.deg2rad(dec)) / pixscale))
H = int(np.ceil((dd.max() - dd.min()) / pixscale))
print('W, H', W, H)
optdict.update(survey=survey)
survey, kwargs = get_runbrick_kwargs(**optdict)
kwargs.update(radec=(ra, dec), width=W, height=H, bands=[ccd.filter])
#if opt.brick is None and opt.radec is None:
run_brick(None, survey, **kwargs)
#hybridPsf=True, normalizePsf=True, rex=True, splinesky=True,
#gaia_stars=True, wise=False, ceres=False,
print('Finished:', Time() - t0)
