def run_psfex(self, surveyname):
from astrometry.util.file import trymakedirs
from legacypipe.survey import get_git_version
sedir = self.survey.get_se_dir()
trymakedirs(self.psffn, dir=True)
primhdr = self.read_image_primary_header()
plver = primhdr.get('PLVER', '')
verstr = get_git_version()
cmds = [
'psfex -c %s -PSF_DIR %s %s' %
(os.path.join(sedir, surveyname + '.psfex'),
os.path.dirname(self.psffn), self.sefn),
'modhead %s LEGPIPEV %s "legacypipe git version"' %
(self.psffn, verstr),
'modhead %s PLVER %s "CP ver of image file"' % (self.psffn, plver)
]
for cmd in cmds:
print(cmd)
rtn = os.system(cmd)
if rtn:
raise RuntimeError('Command failed: %s: return value: %i' %
(cmd, rtn))
def run_one_brick(X):
brick, ibrick, nbricks, plots, kwargs = X
survey = LegacySurveyData()
print()
print()
print('Brick', (ibrick + 1), 'of', nbricks, ':', brick.brickname)
dirnm = os.path.join('depthcuts', brick.brickname[:3])
outfn = os.path.join(dirnm, 'ccds-%s.fits' % brick.brickname)
if os.path.exists(outfn):
print('Exists:', outfn)
return 0
H, W = 3600, 3600
pixscale = 0.262
bands = ['g', 'r', 'z']
# Get WCS object describing brick
targetwcs = wcs_for_brick(brick, W=W, H=H, pixscale=pixscale)
targetrd = np.array([
targetwcs.pixelxy2radec(x, y)
for x, y in [(1, 1), (W, 1), (W, H), (1, H), (1, 1)]
])
gitver = get_git_version()
ccds = survey.ccds_touching_wcs(targetwcs)
if ccds is None:
print('No CCDs actually touching brick')
return 0
print(len(ccds), 'CCDs actually touching brick')
ccds.cut(np.in1d(ccds.filter, bands))
print('Cut on filter:', len(ccds), 'CCDs remain.')
if 'ccd_cuts' in ccds.get_columns():
norig = len(ccds)
ccds.cut(ccds.ccd_cuts == 0)
print(len(ccds), 'of', norig, 'CCDs pass cuts')
else:
print('No CCD cuts')
if len(ccds) == 0:
print('No CCDs left')
return 0
ps = None
if plots:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('depth-%s' % brick.brickname)
splinesky = True
gaussPsf = False
pixPsf = True
do_calibs = False
normalizePsf = True
get_depth_maps = kwargs.pop('get_depth_maps', False)
try:
D = make_depth_cut(survey,
ccds,
bands,
targetrd,
brick,
W,
H,
pixscale,
plots,
ps,
splinesky,
gaussPsf,
pixPsf,
normalizePsf,
do_calibs,
gitver,
targetwcs,
get_depth_maps=get_depth_maps,
**kwargs)
if get_depth_maps:
keep, overlapping, depthmaps = D
else:
keep, overlapping = D
except:
print('Failed to make_depth_cut():')
import traceback
traceback.print_exc()
return -1
print(np.sum(overlapping), 'CCDs overlap the brick')
print(np.sum(keep), 'CCDs passed depth cut')
ccds.overlapping = overlapping
ccds.passed_depth_cut = keep
if not os.path.exists(dirnm):
try:
os.makedirs(dirnm)
except:
pass
if get_depth_maps:
for band, depthmap in depthmaps:
doutfn = os.path.join(dirnm,
'depth-%s-%s.fits' % (brick.brickname, band))
hdr = fitsio.FITSHDR()
# Plug the WCS header cards into these images
targetwcs.add_to_header(hdr)
hdr.delete('IMAGEW')
hdr.delete('IMAGEH')
hdr.add_record(dict(name='EQUINOX', value=2000.))
hdr.add_record(dict(name='FILTER', value=band))
fitsio.write(doutfn, depthmap, header=hdr)
print('Wrote', doutfn)
tmpfn = os.path.join(os.path.dirname(outfn),
'tmp-' + os.path.basename(outfn))
ccds.writeto(tmpfn)
os.rename(tmpfn, outfn)
print('Wrote', outfn)
return 0
def main(args=None):
import os
import datetime
import logging
import numpy as np
from legacypipe.survey import get_git_version
from legacypipe.runbrick import (get_parser, get_runbrick_kwargs,
run_brick, NothingToDoError,
RunbrickError)
print()
print('mpi-runbrick.py starting at', datetime.datetime.now().isoformat())
print('legacypipe git version:', get_git_version())
if args is None:
print('Command-line args:', sys.argv)
cmd = 'python'
for vv in sys.argv:
cmd += ' {}'.format(vv)
print(cmd)
else:
print('Args:', args)
print()
parser = get_parser()
opt = parser.parse_args(args=args)
if opt.brick is None and opt.radec is None:
parser.print_help()
return -1
optdict = vars(opt)
verbose = optdict.pop('verbose')
survey, kwargs = get_runbrick_kwargs(**optdict)
if kwargs in [-1, 0]:
return kwargs
kwargs.update(command_line=' '.join(sys.argv))
if verbose == 0:
lvl = logging.INFO
else:
lvl = logging.DEBUG
global_init(lvl)
# matplotlib config directory, when running in shifter...
for tag in ['CACHE', 'CONFIG']:
if not 'XDG_%s_HOME' % tag in os.environ:
src = os.path.expanduser('~/.%s' % (tag.lower()))
# Read-only?
if os.path.isdir(src) and not (os.access(src, os.W_OK)):
import shutil
import tempfile
tempdir = tempfile.mkdtemp(prefix='%s-' % tag.lower())
os.rmdir(tempdir)
shutil.copytree(src, tempdir,
symlinks=True) #dirs_exist_ok=True)
# astropy config file: if XDG_CONFIG_HOME is set,
# expect to find $XDG_CONFIG_HOME/astropy, not
# ~/.astropy.
if tag == 'CONFIG':
shutil.copytree(os.path.expanduser('~/.astropy'),
os.path.join(tempdir, 'astropy'),
symlinks=True) #dirs_exist_ok=True)
os.environ['XDG_%s_HOME' % tag] = tempdir
if opt.plots:
import matplotlib
matplotlib.use('Agg')
import pylab as plt
plt.figure(figsize=(12, 9))
plt.subplots_adjust(left=0.07,
right=0.99,
bottom=0.07,
top=0.93,
hspace=0.2,
wspace=0.05)
# The "initializer" arg is only available in mpi4py master
pool = MyMPIPool(initializer=global_init, initargs=(lvl, ))
u = int(os.environ.get('OMPI_UNIVERSE_SIZE', '0'))
if u == 0:
u = int(os.environ.get('MPICH_UNIVERSE_SIZE', '0'))
if u == 0:
from mpi4py import MPI
u = MPI.COMM_WORLD.Get_size()
print('Booting up MPI pool with', u, 'workers...')
pool.bootup()
print('Booted up MPI pool.')
pool._processes = u
kwargs.update(pool=pool)
#pool.map(hello, np.arange(128))
rtn = -1
try:
run_brick(opt.brick, survey, **kwargs)
rtn = 0
except NothingToDoError as e:
print()
if hasattr(e, 'message'):
print(e.message)
else:
print(e)
print()
rtn = 0
except RunbrickError as e:
print()
if hasattr(e, 'message'):
print(e.message)
else:
print(e)
print()
rtn = -1
print('Shutting down MPI pool...')
pool.shutdown()
print('Shut down MPI pool')
return rtn
# Setup.py, you suck
#setup_keywords['test_suite'] = 'nose.collector'
#setup_keywords['tests_require'] = ['nose']
#setup_keywords['test_suite'] = 'py/test'
#setup_keywords['test_suite'] = 'run_tests'
#
# Import this module to get __doc__ and __version__.
#
sys.path.insert(int(sys.path[0] == ''), './py')
try:
from importlib import import_module
product = import_module(setup_keywords['name'])
setup_keywords['long_description'] = product.__doc__
from legacypipe.survey import get_git_version
version = get_git_version(os.path.dirname(__file__)).replace('-', '.')
setup_keywords['version'] = version
except ImportError:
#
# Try to get the long description from the README.rst file.
#
if os.path.exists('README.rst'):
with open('README.rst') as readme:
setup_keywords['long_description'] = readme.read()
else:
setup_keywords['long_description'] = ''
setup_keywords['version'] = 'dr3.dev'
#
# Set other keywords for the setup function. These are automated, & should
# setup_keywords['tests_require'] = ['nose']
# setup_keywords['test_suite'] = 'py/test'
# setup_keywords['test_suite'] = 'run_tests'
#
# Import this module to get __doc__ and __version__.
#
sys.path.insert(int(sys.path[0] == ""), "./py")
try:
from importlib import import_module
product = import_module(setup_keywords["name"])
setup_keywords["long_description"] = product.__doc__
from legacypipe.survey import get_git_version
version = get_git_version(os.path.dirname(__file__)).replace("-", ".")
setup_keywords["version"] = version
except ImportError:
#
# Try to get the long description from the README.rst file.
#
if os.path.exists("README.rst"):
with open("README.rst") as readme:
setup_keywords["long_description"] = readme.read()
else:
setup_keywords["long_description"] = ""
setup_keywords["version"] = "dr3.dev"
#
# Set other keywords for the setup function. These are automated, & should
def main():
sbfn = 'skybricks.fits'
SB = fits_table(sbfn)
Bnorth = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/north/survey-bricks-dr9-north.fits.gz'
)
Bsouth = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/south/survey-bricks-dr9-south.fits.gz'
)
Bnorth.cut(Bnorth.survey_primary)
Bsouth.cut(Bsouth.survey_primary)
#Bsouth.cut(Bsouth.dec > -30)
Bnorth.hemi = np.array(['north'] * len(Bnorth))
Bsouth.hemi = np.array(['south'] * len(Bsouth))
B = merge_tables([Bnorth, Bsouth])
# Rough cut the skybricks to those near bricks.
I, J, d = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., nearest=True)
SB.cut(I)
import argparse
parser = argparse.ArgumentParser()
#parser.add_argument('--brick', help='Sky brick name')
parser.add_argument('--minra',
type=float,
help='Cut to a minimum RA range of sky bricks')
parser.add_argument('--maxra',
type=float,
help='Cut to a maximum RA range of sky bricks')
parser.add_argument('--threads',
type=int,
help='Parallelize on this many cores')
opt = parser.parse_args()
if opt.minra:
SB.cut(SB.ra >= opt.minra)
if opt.maxra:
SB.cut(SB.ra <= opt.maxra)
version = get_git_version(os.getcwd())
print('Version string:', version)
# Find bricks near skybricks, as a rough cut.
# (magic 1. degree > hypot(skybrick radius, brick radiu) ~= 0.9)
Inear = match_radec(SB.ra, SB.dec, B.ra, B.dec, 1., indexlist=True)
args = []
k = 0
Isb = []
for isb, (sb, inear) in enumerate(zip(SB, Inear)):
if inear is None:
continue
args.append((k, sb, B[np.array(inear)], version))
k += 1
Isb.append(isb)
print(len(args), 'sky bricks')
SB.cut(np.array(Isb))
if opt.threads:
mp = multiproc(opt.threads)
else:
mp = multiproc()
exist = mp.map(run_one, args)
if opt.minra is None and opt.maxra is None:
exist = np.array(exist)
SB[exist].writeto('skybricks-exist.fits')
return
return
if __name__ == '__main__':
#main()
#B = fits_table('bricks-near.fits')
#SB = fits_table('skybricks.fits')
B = fits_table(
'/global/cfs/cdirs/cosmo/data/legacysurvey/dr9/north/survey-bricks-dr9-north.fits.gz'
)
B.cut(B.dec > 77)
B.hemi = np.array(['north'] * len(B))
SB = fits_table(
'/global/cfs/cdirs/desi/target/skybricks/v3/skybricks-exist.fits')
#I = np.flatnonzero(np.isin(SB.brickname, [
# '2187p340', '2175p340',
#'1500p290', '1507p300', '1519p300',
#'1505p310', '1490p320'
#]))
I = np.flatnonzero(SB.dec >= 78.)
version = get_git_version(os.getcwd())
#version = 4
mp = multiproc(32)
args = []
for i in I:
args.append((0, SB[i], B, version))
mp.map(run_one, args)