def append(self, val):
try:
bkgd, interpStyle, undersampleStyle, approxStyle, approxOrderX, approxOrderY, approxWeighting = val
except TypeError:
bkgd = val
interpStyle = None
undersampleStyle = None
approxStyle = None
approxOrderX = None
approxOrderY = None
approxWeighting = None
# Check to see if the Background is actually a BackgroundMI.
# Such special treatment is not generally a good idea as it is against the whole idea of subclassing.
# However, lsst.afw.math.makeBackground() returns a Background, even though it's really a BackgroundMI
# under the covers. Persistence requires that the type python sees is the actual type under the covers
# (or it will call the wrong python class's python persistence methods).
# The real solution is to not use makeBackground() in python but call the constructor directly;
# however there is already code using makeBackground(), so this is an attempt to assist the user.
subclassed = afwMath.cast_BackgroundMI(bkgd)
if subclassed is not None:
bkgd = subclassed
else:
from lsst.pex.logging import getDefaultLog
getDefaultLog().warn("Unrecognised Background object %s may be unpersistable." % (bkgd,))
bgInfo = (bkgd, interpStyle, undersampleStyle, approxStyle,
approxOrderX, approxOrderY, approxWeighting)
self._backgrounds.append(bgInfo)
def append(self, val):
try:
bkgd, interpStyle, undersampleStyle, approxStyle, approxOrderX, approxOrderY, approxWeighting = val
except TypeError:
bkgd = val
interpStyle = None
undersampleStyle = None
approxStyle = None
approxOrderX = None
approxOrderY = None
approxWeighting = None
# Check to see if the Background is actually a BackgroundMI.
# Such special treatment is not generally a good idea as it is against the whole idea of subclassing.
# However, lsst.afw.math.makeBackground() returns a Background, even though it's really a BackgroundMI
# under the covers. Persistence requires that the type python sees is the actual type under the covers
# (or it will call the wrong python class's python persistence methods).
# The real solution is to not use makeBackground() in python but call the constructor directly;
# however there is already code using makeBackground(), so this is an attempt to assist the user.
subclassed = afwMath.cast_BackgroundMI(bkgd)
if subclassed is not None:
bkgd = subclassed
else:
from lsst.pex.logging import getDefaultLog
getDefaultLog().warn(
"Unrecognised Background object %s may be unpersistable." %
(bkgd, ))
bgInfo = (bkgd, interpStyle, undersampleStyle, approxStyle,
approxOrderX, approxOrderY, approxWeighting)
self._backgrounds.append(bgInfo)
def getEnvironmentPackages():
"""Provide a dict of products and their versions from the environment
We use EUPS to determine the version of certain products (those that don't provide
a means to determine the version any other way) and to check if uninstalled packages
are being used. We only report the product/version for these packages.
"""
try:
from eups import Eups
from eups.Product import Product
except:
from lsst.pex.logging import getDefaultLog
getDefaultLog().warn("Unable to import eups, so cannot determine package versions from environment")
return {}
# Cache eups object since creating it can take a while
global _eups
if not _eups:
_eups = Eups()
products = _eups.findProducts(tags=["setup"])
# Get versions for things we can't determine via runtime mechanisms
# XXX Should we just grab everything we can, rather than just a predetermined set?
packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
# The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled
# code, so the version could be different than what's being reported by the runtime environment (because
# we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils
# probably doesn't check to see if the repo is clean).
for prod in products:
if not prod.version.startswith(Product.LocalVersionPrefix):
continue
ver = prod.version
gitDir = os.path.join(prod.dir, ".git")
if os.path.exists(gitDir):
# get the git revision and an indication if the working copy is clean
revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
"--patch"]
try:
rev = subprocess.check_output(revCmd).decode().strip()
diff = subprocess.check_output(diffCmd)
except:
ver += "@GIT_ERROR"
else:
ver += "@" + rev
if diff:
ver += "+" + hashlib.md5(diff).hexdigest()
else:
ver += "@NO_GIT"
packages[prod.name] = ver
return packages
class MO(object):
"""Measure the sources on a frame"""
def __init__(self, display = False, rhs = None):
self.display = display
self.gas = None
if rhs:
try:
self.exposure = rhs.exposure
self.gas = rhs.gas
self.pixscale = rhs.pixscale
self.psf = rhs.psf
self.sourceList = rhs.sourceList
self.XY0 = rhs.XY0
try:
self.psfImage = rhs.psfImage
except AttributeError:
pass
except AttributeError, e:
raise RuntimeError, ("Unexpected rhs: %s (%s)" % (rhs, e))
self.display = display
# set up a log
self.scriptLog = pexLog.getDefaultLog()
self.scriptLog.setThreshold(pexLog.Log.WARN)
self.log = pexLog.Log(self.scriptLog, "measureSources")
def __init__(self, config):
"""
@param[in] config: instance of PolypixPsfDeterminerConfig
"""
self.config = config
# N.b. name of component is meas.algorithms.psfDeterminer so you can turn on psf debugging
# independent of which determiner is active
self.debugLog = pexLog.Debug("meas.algorithms.psfDeterminer")
self.warnLog = pexLog.Log(pexLog.getDefaultLog(), "meas.algorithms.psfDeterminer")
def testLogging(self):
# test a simple message to the default log
dlog = log.getDefaultLog()
dlog.log(log.Log.WARN, "this is a warning")
# now let's create our own root log
logger = log.ScreenLog(True)
# test creation of child log
tlog = log.Log(logger, "test")
tlog.log(log.Log.INFO, "I like your hat")
# test that "format", "infof", etc are ignored by swig.
self.assertFalse(hasattr(tlog, 'format'))
self.assertFalse(hasattr(tlog, 'debugf'))
self.assertFalse(hasattr(tlog, 'infof'))
self.assertFalse(hasattr(tlog, 'warnf'))
self.assertFalse(hasattr(tlog, 'fatalf'))
# test shortcut methods
tlog.logdebug("Shortcut debug")
tlog.info("Shortcut info")
tlog.warn("Shortcut warn")
tlog.fatal("Shortcut fatal")
# test threshold filtering
tlog.setThreshold(log.Log.WARN)
tlog.log(log.Log.INFO, "I like your gloves") # // shouldn't see this
# test the persistance of threshold levels
tlog = log.Log(logger, "test")
tlog.log(log.Log.INFO, "I like your shoes") # // shouldn't see this
tlog.setThreshold(log.Log.DEBUG)
tlog.log(log.Log.INFO, "I said, I like your shoes")
# test descendent log and ancestor's control of threshold
tgclog = log.Log(tlog,
"grand.child") # // name is now "test.grand.child"
tgclog.log(log.Log.INFO, "Let's play")
tlog.setThreshold(log.Log.FATAL)
tgclog.log(log.Log.INFO, "You go first")
# test streaming
log.LogRec(tgclog, log.Log.FATAL) << "help: I've fallen" << log.Prop(
"NODE", 5) << "& I can't get up" << log.endr
tmp = log.Prop("NODE", 5)
log.LogRec(
tgclog, log.Log.FATAL
) << "help: I've fallen" << tmp << "& I can't get up" << log.endr
# test flushing on delete
log.Rec(tgclog, log.Log.FATAL) << "never mind"
log.Rec(tgclog, log.Log.DEBUG) << "testing" << log.endr
def importMatplotlib():
"""!Import matplotlib.pyplot when needed, warn if not available.
@return the imported module if available, else False.
"""
try:
import matplotlib.pyplot as plt
return plt
except ImportError as err:
log = pexLog.Log(pexLog.getDefaultLog(), 'lsst.ip.diffim.utils',
pexLog.INFO)
log.warn('Unable to import matplotlib: %s' % err)
return False
def importMatplotlib():
"""!Import matplotlib.pyplot when needed, warn if not available.
@return the imported module if available, else False.
"""
try:
import matplotlib.pyplot as plt
return plt
except ImportError as err:
log = pexLog.Log(pexLog.getDefaultLog(),
'lsst.ip.diffim.utils', pexLog.INFO)
log.warn('Unable to import matplotlib: %s' % err)
return False
def testOverrides(self):
"""Test config and log override
"""
config = TestTask.ConfigClass()
config.floatField = -99.9
defLog = pexLog.getDefaultLog()
log = pexLog.Log(defLog, "cmdLineTask")
retVal = TestTask.parseAndRun(
args=[DataPath, "--output", self.outPath, "--id", "visit=2"],
config = config,
log = log
)
self.assertEquals(retVal.parsedCmd.config.floatField, -99.9)
self.assertTrue(retVal.parsedCmd.log is log)
def testLogging(self):
# test a simple message to the default log
dlog = log.getDefaultLog()
dlog.log(log.Log.WARN, "this is a warning")
# now let's create our own root log
logger = log.ScreenLog(True)
# test creation of child log
tlog = log.Log(logger, "test")
tlog.log(log.Log.INFO, "I like your hat")
# test that "format", "infof", etc are ignored by swig.
self.assertFalse(hasattr(tlog, 'format'))
self.assertFalse(hasattr(tlog, 'debugf'))
self.assertFalse(hasattr(tlog, 'infof'))
self.assertFalse(hasattr(tlog, 'warnf'))
self.assertFalse(hasattr(tlog, 'fatalf'))
# test shortcut methods
tlog.logdebug("Shortcut debug")
tlog.info("Shortcut info")
tlog.warn("Shortcut warn")
tlog.fatal("Shortcut fatal")
# test threshold filtering
tlog.setThreshold(log.Log.WARN)
tlog.log(log.Log.INFO, "I like your gloves") # // shouldn't see this
# test the persistance of threshold levels
tlog = log.Log(logger, "test")
tlog.log(log.Log.INFO, "I like your shoes") # // shouldn't see this
tlog.setThreshold(log.Log.DEBUG)
tlog.log(log.Log.INFO, "I said, I like your shoes")
# test descendent log and ancestor's control of threshold
tgclog = log.Log(tlog, "grand.child") # // name is now "test.grand.child"
tgclog.log(log.Log.INFO, "Let's play")
tlog.setThreshold(log.Log.FATAL)
tgclog.log(log.Log.INFO, "You go first")
# test streaming
log.LogRec(tgclog, log.Log.FATAL) << "help: I've fallen" << log.Prop("NODE", 5) << "& I can't get up" << log.endr;
tmp = log.Prop("NODE", 5)
log.LogRec(tgclog, log.Log.FATAL) << "help: I've fallen" << tmp << "& I can't get up" << log.endr;
# test flushing on delete
log.Rec(tgclog, log.Log.FATAL) << "never mind"
log.Rec(tgclog, log.Log.DEBUG) << "testing" << log.endr;
def __init__(self, config=None, name=None, parentTask=None, log=None):
"""!Create a Task
@param[in] config configuration for this task (an instance of self.ConfigClass,
which is a task-specific subclass of lsst.pex.config.Config), or None. If None:
- If parentTask specified then defaults to parentTask.config.\<name>
- If parentTask is None then defaults to self.ConfigClass()
@param[in] name brief name of task, or None; if None then defaults to self._DefaultName
@param[in] parentTask the parent task of this subtask, if any.
- If None (a top-level task) then you must specify config and name is ignored.
- If not None (a subtask) then you must specify name
@param[in] log pexLog log; if None then the default is used;
in either case a copy is made using the full task name.
@throw RuntimeError if parentTask is None and config is None.
@throw RuntimeError if parentTask is not None and name is None.
@throw RuntimeError if name is None and _DefaultName does not exist.
"""
self.metadata = dafBase.PropertyList()
self._parentTask = parentTask
if parentTask != None:
if name is None:
raise RuntimeError("name is required for a subtask")
self._name = name
self._fullName = parentTask._computeFullName(name)
if config == None:
config = getattr(parentTask.config, name)
self._taskDict = parentTask._taskDict
self.log = pexLog.Log(parentTask.log, name)
else:
if name is None:
name = getattr(self, "_DefaultName", None)
if name is None:
raise RuntimeError("name is required for a task unless it has attribute _DefaultName")
name = self._DefaultName
self._name = name
self._fullName = self._name
if config == None:
config = self.ConfigClass()
self._taskDict = dict()
if log is None:
log = pexLog.getDefaultLog()
self.log = pexLog.Log(log, self._fullName)
self.config = config
self._display = lsstDebug.Info(self.__module__).display
self._taskDict[self._fullName] = self
def _poolFunctionWrapper(function, arg):
"""Wrapper around function to catch exceptions that don't inherit from Exception
Such exceptions aren't caught by multiprocessing, which causes the slave
process to crash and you end up hitting the timeout.
"""
try:
return function(arg)
except Exception:
raise # No worries
except:
# Need to wrap the exception with something multiprocessing will recognise
cls, exc, tb = sys.exc_info()
log = getDefaultLog()
log.warn("Unhandled exception %s (%s):\n%s" % (cls.__name__, exc, traceback.format_exc()))
raise Exception("Unhandled exception: %s (%s)" % (cls.__name__, exc))
def __call__(self, args):
"""!Run the Task on a single target.
This default implementation assumes that the 'args' is a tuple
containing a data reference and a dict of keyword arguments.
@warning if you override this method and wish to return something when
doReturnResults is false, then it must be picklable to support
multiprocessing and it should be small enough that pickling and
unpickling do not add excessive overhead.
@param args Arguments for Task.run()
@return:
- None if doReturnResults false
- A pipe_base Struct containing these fields if doReturnResults true:
- dataRef: the provided data reference
- metadata: task metadata after execution of run
- result: result returned by task run, or None if the task fails
"""
dataRef, kwargs = args
if self.log is None:
self.log = getDefaultLog()
if hasattr(dataRef, "dataId"):
self.log.addLabel(str(dataRef.dataId))
elif isinstance(dataRef, (list, tuple)):
self.log.addLabel(str([ref.dataId for ref in dataRef if hasattr(ref, "dataId")]))
task = self.makeTask(args=args)
result = None # in case the task fails
if self.doRaise:
result = task.run(dataRef, **kwargs)
else:
try:
result = task.run(dataRef, **kwargs)
except Exception, e:
# don't use a try block as we need to preserve the original exception
if hasattr(dataRef, "dataId"):
task.log.fatal("Failed on dataId=%s: %s" % (dataRef.dataId, e))
elif isinstance(dataRef, (list, tuple)):
task.log.fatal("Failed on dataId=[%s]: %s" %
(",".join([str(_.dataId) for _ in dataRef]), e))
else:
task.log.fatal("Failed on dataRef=%s: %s" % (dataRef, e))
if not isinstance(e, TaskError):
traceback.print_exc(file=sys.stderr)
def __init__(self, filenameList, healpix, nside):
"""!Constructor
@param filenameList List of filenames; first is the multiindex, then
follows the individual index files
@param healpix Healpix number
@param nside Healpix nside
"""
if len(filenameList) < 2:
raise RuntimeError("Insufficient filenames provided for multiindex (%s): expected >= 2" %
(filenameList,))
self._filenameList = filenameList
self._healpix = int(healpix)
self._nside = int(nside)
self._mi = None
self._loaded = False
self.log = getDefaultLog()
def __init__(self,
script,
importList=None,
command=None,
resourceList=None,
queue=None):
"""Initialisation
@param script Text of script to execute
@param importList List of imports; may be specified as a 2-tuple for 'import X as Y'
@param command Command to run to submit to queue ("qsub -V")
@param resourceList List of resources for PBS
@param queue Name of queue
"""
# Remove common indentation
lines = re.split("\n", script)
exemplar = None # First non-blank line
for line in lines:
if re.search("\S", line):
exemplar = line
break
if exemplar is None:
raise RuntimeError("Empty script provided.")
match = re.match("(\s+)", exemplar)
if match:
indent = match.group(0) # Indentation used
newLines = []
for line in lines:
if not re.search("\S", line):
continue
newLine = re.sub("^" + indent, "", line)
if newLine is None:
raise RuntimeError("Inconsistent indentation in script: " +
script)
newLines.append(newLine)
script = "\n".join(newLines)
self.script = script
self.importList = importList
self.command = command if command is not None else "qsub -V"
self.resourceList = resourceList
self.queue = queue
self.log = pexLog.Log(pexLog.getDefaultLog(), "PbsQueue")
return
def __init__(
self, # ReadWrite
mappers, # Mapper or mapper class to use
ccdKeys, # Data keywords required to specify a CCD
fileKeys=None, # Data keywords required to specify a file
config=None, # Configuration
):
"""Initialisation
@param mapper Data mapper (class or instance) for persistence
@param config Configuration (for instantiating mapper)
"""
# if we got a list, it contains [inMapper, outMapper]
if isinstance(mappers, list) and len(mappers) == 2:
inMapper, outMapper = mappers
# if we got a mapper, use it for both input and output
elif (isinstance(mappers, dafPersist.Mapper)
or issubclass(mappers, dafPersist.Mapper)):
inMapper, outMapper = mappers, mappers
# punt
else:
raise RuntimeError(
"'mapper' must be a dafPersist.Mapper (or derived from), or a list containing two of them (in and out)."
)
self.log = pexLog.Log(pexLog.getDefaultLog(), "ReadWrite")
self.inMapper = initMapper(inMapper, config, self.log, inMap=True)
self.ibf = dafPersist.ButlerFactory(mapper=self.inMapper)
self.inButler = self.ibf.create()
self.outMapper = initMapper(outMapper, config, self.log, inMap=False)
self.obf = dafPersist.ButlerFactory(mapper=self.outMapper)
self.outButler = self.obf.create()
self.ccdKeys = ccdKeys
if fileKeys is None:
fileKeys = list(ccdKeys)
if isinstance(fileKeys, basestring):
fileKeys = [fileKeys]
self.fileKeys = fileKeys
return
def __init__(self, rerun=0, basedir='.', **kwargs):
Mapper.__init__(self)
print 'TractorMapper(): ignoring kwargs', kwargs
self.basedir = basedir
self.rerun = rerun
self.log = pexLog.Log(pexLog.getDefaultLog(), 'TractorMapper')
indir = os.path.join(self.basedir, 't%(visit)04i')
outdir = os.path.join(indir, 'rr%(rerun)04i')
self.filenames = { 'outdir': (outdir, None, None),
'visitim': (os.path.join(indir, 't.fits'), #'t_img.fits'), #img.fits'),
'lsst.afw.image.ExposureF', 'ExposureF'),
'psf': (os.path.join(outdir, 'psf.boost'),
'lsst.afw.detection.Psf', 'Psf'),
'src': (os.path.join(outdir, 'src.boost'),
# dare to dream / keep dreaming
#os.path.join(outdir, 'src.fits'),
# htf did this work before?
#'lsst.afw.detection.Source', 'Source'),
'lsst.afw.detection.PersistableSourceVector',
'PersistableSourceVector'),
'bb': (os.path.join(outdir, 'bb.pickle'),
None, None),
'pyfoots': (os.path.join(outdir, 'foots.pickle'),
None, None),
'footprints': (os.path.join(outdir, 'foots.boost'),
'lsst.afw.detection.FootprintList',
'FootprintList'),
'truesrc': (os.path.join(indir, 'srcs.fits'),
None, None),
}
'''
for datasetType in ["raw", "bias", "dark", "flat", "fringe",
"postISR", "postISRCCD", "sdqaAmp", "sdqaCcd",
"icSrc", "icMatch", "visitim", "psf", "apCorr", "calexp", "src",
"sourceHist", "badSourceHist", "source", "badSource",
"invalidSource", "object", "badObject"]:
'''
self.keys = ['visit', 'filter']
def __init__(self, config=None, log=None):
self.config = config
if log is None: log = pexLog.getDefaultLog()
self.log = pexLog.Log(log, self.__class__.__name__)
self._display = lsstDebug.Info(__name__).display
def testAlgorithms(self):
"""Test that we can instantiate and use algorithms"""
config = measAlg.SourceMeasurementConfig()
config.algorithms.names = measAlg.AlgorithmRegistry.all.keys()
config.algorithms.names.discard(config.centroider.name)
config.doReplaceWithNoise = False
config.algorithms.names.discard("flux.peakLikelihood")
if False:
log = pexLog.getDefaultLog()
log.setThreshold(log.DEBUG)
schema = afwTable.SourceTable.makeMinimalSchema()
task = measAlg.SourceMeasurementTask(schema, config=config)
catalog = afwTable.SourceCatalog(schema)
source = catalog.addNew()
source.set("id", 12345)
size = 256
xStar, yStar = 65.432, 76.543
width = 3.21
x0, y0 = 12345, 54321
x, y = numpy.indices((size, size))
im = afwImage.MaskedImageF(afwGeom.ExtentI(size, size))
im.setXY0(afwGeom.Point2I(x0, y0))
im.getVariance().set(1.0)
arr = im.getImage().getArray()
arr[y, x] = numpy.exp(-0.5 * ((x - xStar) ** 2 + (y - yStar) ** 2) / width ** 2)
psf = testLib.makeTestPsf(im)
exp = afwImage.makeExposure(im)
exp.setPsf(psf)
exp.setXY0(afwGeom.Point2I(x0, y0))
scale = 1.0e-5
wcs = afwImage.makeWcs(
afwCoord.Coord(0.0 * afwGeom.degrees, 0.0 * afwGeom.degrees),
afwGeom.Point2D(0.0, 0.0),
scale,
0.0,
0.0,
scale,
)
exp.setWcs(wcs)
point = afwGeom.Point2I(int(xStar + x0), int(yStar + y0))
bbox = im.getBBox()
bbox.shift(afwGeom.Extent2I(x0, y0))
foot = afwDetection.Footprint(point, width, bbox)
foot.addPeak(point.getX(), point.getY(), 1.0)
afwDetection.setMaskFromFootprint(exp.getMaskedImage().getMask(), foot, 1)
source.setFootprint(foot)
if display:
ds9.mtv(exp, frame=1)
task.run(exp, catalog)
for alg in config.algorithms:
flagName = alg + ".flags"
if False:
print(
alg,
source.get(flagName) if flagName in schema else None,
source.get(alg) if alg in schema else None,
)
elif flagName in schema:
self.assertFalse(source.get(alg + ".flags"))
"""test simple use of the Log facility."""
"""
testLog
Run with:
python testLog
"""
import lsst.utils.tests as tests
import lsst.pex.logging as log
if __name__ == "__main__":
# test a simple message to the default log
dlog = log.getDefaultLog()
dlog.log(log.Log.WARN, "this is a warning")
# now let's create our own root log
logger = log.ScreenLog(True)
# test creation of child log
tlog = log.Log(logger, "test")
tlog.log(log.Log.INFO, "I like your hat")
# test that "format", "infof", etc are ignored by swig.
assert(not hasattr(tlog, 'format'))
assert(not hasattr(tlog, 'debugf'))
assert(not hasattr(tlog, 'infof'))
assert(not hasattr(tlog, 'warnf'))
assert(not hasattr(tlog, 'fatalf'))
def run(visit, rerun, config):
mapper = getMapper()
dataId = { 'visit': visit, 'rerun': rerun }
rrdir = mapper.getPath('outdir', dataId)
if not os.path.exists(rrdir):
print 'Creating directory for ouputs:', rrdir
os.makedirs(rrdir)
else:
print 'Output directory:', rrdir
io = pipReadWrite.ReadWrite(mapper, ['visit'], config=config)
#ccdProc = pipProcCcd.ProcessCcd(config=config, Isr=NullISR, Calibrate=MyCalibrate)
#raws = io.readRaw(dataId)
#detrends = io.detrends(dataId, config)
print 'Reading exposure'
#exposure = io.read('visitim', dataId)
detrends = []
exposure = io.inButler.get('visitim', dataId)
print 'exposure is', exposure
print 'size', exposure.getWidth(), 'x', exposure.getHeight()
# debug
mi = exposure.getMaskedImage()
#img = mi.getImage()
#var = mi.getVariance()
#print 'var at 90,100 is', var.get(90,100)
#print 'img at 90,100 is', img.get(90,100)
#print 'wcs is', exposure.getWcs()
wcs = exposure.getWcs()
assert wcs
#print 'ccdProc.run()...'
# raws = [exposure]
#exposure, psf, apcorr, brightSources, sources, matches, matchMeta = ccdProc.run(raws, detrends)
print 'Calibrate()...'
log = pexLog.getDefaultLog()
cal = MyCalibrate(config=config, log=log, Photometry=MyPhotometry)
psf,sources,footprints = cal.run2(exposure)
print 'Photometry()...'
phot = pipPhot.Photometry(config=config, log=log)
sources, footprints = phot.run(exposure, psf)
print 'sources:', len(sources)
for s in sources:
print ' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(), s.getIxx(), s.getIyy(), s.getIxy()
print 'footprints:', footprints
# oh yeah, baby!
fps = footprints.getFootprints()
print len(fps)
bb = []
for f in fps:
print ' Footprint', f
print ' ', f.getBBox()
bbox = f.getBBox()
bb.append((bbox.getMinX(), bbox.getMinY(), bbox.getMaxX(), bbox.getMaxY()))
print ' # peaks:', len(f.getPeaks())
for p in f.getPeaks():
print ' Peak', p
#print 'psf', psf
#print 'sources', sources
#print 'footprints', footprints
#psf, apcorr, brightSources, matches, matchMeta = self.calibrate(exposure, defects=defects)
#if self.config['do']['phot']:
# sources, footprints = self.phot(exposure, psf, apcorr, wcs=exposure.getWcs())
#psf, wcs = self.fakePsf(exposure)
#sources, footprints = self.phot(exposure, psf)
#sources = self.rephot(exposure, footprints, psf, apcorr=apcorr)
#model = calibrate['model']
#fwhm = calibrate['fwhm'] / wcs.pixelScale()
#size = calibrate['size']
# psf = afwDet.createPsf(model, size, size, fwhm/(2*math.sqrt(2*math.log(2))))
#print 'done!'
print 'writing output...'
io.write(dataId, psf=psf, sources=sources)
print 'done!'
print 'Writing bounding-boxes...'
io.outButler.put(bb, 'bb', dataId)
#print 'Writing footprints...'
#io.outButler.put(fps, 'footprints', dataId)
# serialize a python version of footprints & peaks
pyfoots = footprintsToPython(fps)
print 'Writing py footprints...'
io.outButler.put(pyfoots, 'pyfoots', dataId)
return bb
# You should have received a copy of the LSST License Statement and
# the GNU General Public License along with this program. If not,
# see <http://www.lsstcorp.org/LegalNotices/>.
#
"""
@brief Demonstrate the use of the logging framework from Python
"""
import lsst.pex.logging as log
from lsst.pex.logging import Rec, Prop, endr
if __name__ == "__main__":
# in any script or function where you want to log message, the first
# thing you should do is get create a Log from the DefaultLog
mylog = log.Log(log.getDefaultLog(), "myapp.myfunc")
# simple message can be sent with a function. The first argument is
# the level of message; choices are: DEBUG, INFO, WARN, FATAL.
mylog.log(mylog.INFO, "this is a simple message")
# If you want to send multiple messages and/or properties all in the
# same message, you can use the shift operator. Be sure to end the
# message with "endr"
Rec(mylog, mylog.WARN) << "ouch!" << Prop("NODE", 5) \
<< "something bad happened!" << endr
# Normally properties are not printed to the screen. To see these, we'll
# turn them on now.
#
# Outside the pipeline framework, the default logger is a ScreenLog.
def readFits(fileName, hdu=0, flags=0):
"""Read a ds9 region file, returning a ObjectMaskCatalog object
This method is called "readFits" to fool the butler. The corresponding mapper entry looks like
brightObjectMask: {
template: "deepCoadd/BrightObjectMasks/%(tract)d/BrightObjectMask-%(tract)d-%(patch)s-%(filter)s.reg"
python: "lsst.obs.subaru.objectMasks.ObjectMaskCatalog"
persistable: "PurePythonClass"
storage: "FitsCatalogStorage"
}
and this is the only way I know to get it to read a random file type, in this case a ds9 region file
This method expects to find files named as BrightObjectMask-%(tract)d-%(patch)s-%(filter)s.reg
The files should be structured as follows:
# Description of catalogue as a comment
# CATALOG: catalog-id-string
# TRACT: 0
# PATCH: 5,4
# FILTER: HSC-I
wcs; fk5
circle(RA, DEC, RADIUS) # ID: 1
The commented lines must be present, with the relevant fields such as tract patch and filter filled
in. The coordinate system must be listed as above. Each patch is specified as a circle, with an RA,
DEC, and Radius specified in decimal degrees. Only circles are supported as region definitions
currently.
"""
log = pexLog.getDefaultLog().createChildLog("ObjectMaskCatalog")
brightObjects = ObjectMaskCatalog()
checkedWcsIsFk5 = False
with open(fileName) as fd:
for lineNo, line in enumerate(fd.readlines(), 1):
line = line.rstrip()
if re.search(r"^\s*#", line):
#
# Parse any line of the form "# key : value" and put them into the metadata.
#
# The medatdata values must be defined as outlined in the above docstring
#
# The value of these three keys will be checked,
# so get them right!
#
mat = re.search(
r"^\s*#\s*([a-zA-Z][a-zA-Z0-9_]+)\s*:\s*(.*)", line)
if mat:
key, value = mat.group(1).lower(), mat.group(2)
if key == "tract":
value = int(value)
brightObjects.table.getMetadata().set(key, value)
line = re.sub(r"^\s*#.*", "", line)
if not line:
continue
if re.search(r"^\s*wcs\s*;\s*fk5\s*$", line, re.IGNORECASE):
checkedWcsIsFk5 = True
continue
# This regular expression parses the regions file for each region to be masked,
# with the format as specified in the above docstring.
mat = re.search(
r"^\s*circle(?:\s+|\s*\(\s*)"
"(\d+(?:\.\d*)([d]*))"
"(?:\s+|\s*,\s*)"
"([+-]?\d+(?:\.\d*)([d]*))"
"(?:\s+|\s*,\s*)"
"(\d+(?:\.\d*))([d'\"]*)"
"(?:\s*|\s*\)\s*)"
"\s*#\s*ID:\s*(\d+)"
"\s*$", line)
if mat:
ra, raUnit, dec, decUnit, radius, radiusUnit, _id = mat.groups(
)
_id = int(_id)
ra = convertToAngle(ra, raUnit, "ra", fileName, lineNo)
dec = convertToAngle(dec, decUnit, "dec", fileName, lineNo)
radius = convertToAngle(radius, radiusUnit, "radius",
fileName, lineNo)
rec = brightObjects.addNew()
# N.b. rec["coord"] = Coord is not supported, so we have to use the setter
rec["id"] = _id
rec.setCoord(afwCoord.Fk5Coord(ra, dec))
rec["radius"] = radius
else:
log.warn("Unexpected line \"%s\" at %s:%d" %
(line, fileName, lineNo))
if not checkedWcsIsFk5:
raise RuntimeError("Expected to see a line specifying an fk5 wcs")
# This makes the deep copy contiguous in memory so that a ColumnView can be exposed to Numpy
brightObjects._catalog = brightObjects._catalog.copy(True)
return brightObjects
def __init__(self, label, rerun, cameraInfo, qaDataUtils, **kwargs):
"""
@param label The name of this data set
@param rerun The rerun to retrieve
@param cameraInfo A cameraInfo object containing specs on the camera
"""
self.label = label
self.rerun = rerun
self.cameraInfo = cameraInfo
self.dataInfo = self.cameraInfo.dataInfo
self.log = kwargs.get('log', pexLog.getDefaultLog())
self.dataIdNames = []
self.dataIdDiscrim = []
for array in self.dataInfo:
dataIdName, dataIdDiscrim = array
self.dataIdNames.append(dataIdName)
self.dataIdDiscrim.append(dataIdDiscrim)
self.initCache()
self.loadDepth = 0
self.lastPrint = None
self.t0 = []
self.brokenDataIdList = []
self.ccdConvention = 'ccd'
if self.cameraInfo.name == 'lsstSim':
self.ccdConvention = 'sensor'
if self.cameraInfo.name == 'sdss':
self.ccdConvention = 'camcol'
refCatObj = pqaSource.RefCatalog()
self.k_rPsf = refCatObj.keyDict['PsfFlux']
self.k_rAp = refCatObj.keyDict['ApFlux']
self.k_rMod = refCatObj.keyDict['ModelFlux']
self.k_rInst = refCatObj.keyDict['InstFlux']
self.k_rPsfE = refCatObj.keyDict['PsfFluxErr']
self.k_rApE = refCatObj.keyDict['ApFluxErr']
self.k_rModE = refCatObj.keyDict['ModelFluxErr']
self.k_rInstE = refCatObj.keyDict['InstFluxErr']
self.k_rRa = refCatObj.keyDict['Ra']
self.k_rDec = refCatObj.keyDict['Dec']
catObj = pqaSource.Catalog(qaDataUtils=qaDataUtils)
self.k_x = catObj.keyDict['XAstrom']
self.k_y = catObj.keyDict['YAstrom']
self.k_Ra = catObj.keyDict['Ra']
self.k_Dec = catObj.keyDict['Dec']
self.k_Psf = catObj.keyDict['PsfFlux']
self.k_Ap = catObj.keyDict['ApFlux']
self.k_Mod = catObj.keyDict['ModelFlux']
self.k_Inst = catObj.keyDict['InstFlux']
self.k_PsfE = catObj.keyDict['PsfFluxErr']
self.k_ApE = catObj.keyDict['ApFluxErr']
self.k_ModE = catObj.keyDict['ModelFluxErr']
self.k_InstE = catObj.keyDict['InstFluxErr']
self.k_ext = catObj.keyDict['Extendedness']
self.k_intc = catObj.keyDict["FlagPixInterpCen"]
self.k_satc = catObj.keyDict["FlagPixSaturCen"]
self.k_edg = catObj.keyDict["FlagPixEdge"]
self.k_neg = catObj.keyDict['FlagNegative']
self.k_bad = catObj.keyDict["FlagBadCentroid"]
self.k_ixx = catObj.keyDict['Ixx']
self.k_iyy = catObj.keyDict['Iyy']
self.k_ixy = catObj.keyDict['Ixy']
self.k_nchild = catObj.keyDict['deblend_nchild']
def __init__(self, root, log=None):
if log is None:
log = getDefaultLog()
self.log = log
self.root = root
self._butler = None
def selectStars(self, exposure, catalog, matches=None):
"""Return a list of PSF candidates that represent likely stars
A list of PSF candidates may be used by a PSF fitter to construct a PSF.
@param[in] exposure: the exposure containing the sources
@param[in] catalog: a SourceCatalog containing sources that may be stars
@param[in] matches: astrometric matches; ignored by this star selector
@return psfCandidateList: a list of PSF candidates.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(
__name__).displayExposure # display the Exposure + spatialCells
plotMagSize = lsstDebug.Info(
__name__).plotMagSize # display the magnitude-size relation
dumpData = lsstDebug.Info(
__name__).dumpData # dump data to pickle file?
# create a log for my application
logger = pexLogging.Log(pexLogging.getDefaultLog(),
"meas.algorithms.objectSizeStarSelector")
detector = exposure.getDetector()
distorter = None
xy0 = afwGeom.Point2D(0, 0)
if not detector is None:
cPix = detector.getCenterPixel()
detSize = detector.getSize()
xy0.setX(cPix.getX() - int(0.5 * detSize.getMm()[0]))
xy0.setY(cPix.getY() - int(0.5 * detSize.getMm()[1]))
distorter = detector.getDistortion()
#
# Look at the distribution of stars in the magnitude-size plane
#
flux = catalog.get(self._sourceFluxField)
xx = numpy.empty(len(catalog))
xy = numpy.empty_like(xx)
yy = numpy.empty_like(xx)
for i, source in enumerate(catalog):
Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
if distorter:
xpix, ypix = source.getX() + xy0.getX(), source.getY(
) + xy0.getY()
p = afwGeom.Point2D(xpix, ypix)
m = distorter.undistort(p, geomEllip.Quadrupole(Ixx, Iyy, Ixy),
detector)
Ixx, Ixy, Iyy = m.getIxx(), m.getIxy(), m.getIyy()
xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
width = numpy.sqrt(xx + yy)
bad = reduce(lambda x, y: numpy.logical_or(x, catalog.get(y)),
self._badFlags, False)
bad = numpy.logical_or(bad, flux < self._fluxMin)
bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
bad = numpy.logical_or(bad, width < self._widthMin)
bad = numpy.logical_or(bad, width > self._widthMax)
if self._fluxMax > 0:
bad = numpy.logical_or(bad, flux > self._fluxMax)
good = numpy.logical_not(bad)
if not numpy.any(good):
raise RuntimeError(
"No objects passed our cuts for consideration as psf stars")
mag = -2.5 * numpy.log10(flux[good])
width = width[good]
#
# Look for the maximum in the size histogram, then search upwards for the minimum that separates
# the initial peak (of, we presume, stars) from the galaxies
#
if dumpData:
import os, cPickle as pickle
_ii = 0
while True:
pickleFile = os.path.expanduser(
os.path.join("~", "widths-%d.pkl" % _ii))
if not os.path.exists(pickleFile):
break
_ii += 1
with open(pickleFile, "wb") as fd:
pickle.dump(mag, fd, -1)
pickle.dump(width, fd, -1)
centers, clusterId = _kcenters(width, nCluster=4, useMedian=True)
if display and plotMagSize and pyplot:
fig = plot(mag,
width,
centers,
clusterId,
marker="+",
markersize=3,
markeredgewidth=None,
ltype=':',
clear=True)
else:
fig = None
clusterId = _improveCluster(width, centers, clusterId)
if display and plotMagSize and pyplot:
plot(mag,
width,
centers,
clusterId,
marker="x",
markersize=3,
markeredgewidth=None)
stellar = (clusterId == 0)
#
# We know enough to plot, if so requested
#
frame = 0
if fig:
if display and displayExposure:
ds9.mtv(exposure.getMaskedImage(),
frame=frame,
title="PSF candidates")
global eventHandler
eventHandler = EventHandler(fig.get_axes()[0],
mag,
width,
catalog.getX()[good],
catalog.getY()[good],
frames=[frame])
fig.show()
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
while True:
try:
reply = raw_input(
"continue? [c h(elp) q(uit) p(db)] ").strip()
except EOFError:
reply = "y"
if reply:
if reply[0] == "h":
print """\
We cluster the points; red are the stellar candidates and the other colours are other clusters.
Points labelled + are rejects from the cluster (only for cluster 0).
At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in ds9.
"""
elif reply[0] == "p":
import pdb
pdb.set_trace()
elif reply[0] == 'q':
sys.exit(1)
else:
break
if display and displayExposure:
mi = exposure.getMaskedImage()
with ds9.Buffering():
for i, source in enumerate(catalog):
if good[i]:
ctype = ds9.GREEN # star candidate
else:
ctype = ds9.RED # not star
ds9.dot("+",
source.getX() - mi.getX0(),
source.getY() - mi.getY0(),
frame=frame,
ctype=ctype)
#
# Time to use that stellar classification to generate psfCandidateList
#
with ds9.Buffering():
psfCandidateList = []
for isStellar, source in zip(
stellar, [s for g, s in zip(good, catalog) if g]):
if not isStellar:
continue
try:
psfCandidate = algorithmsLib.makePsfCandidate(
source, exposure)
# The setXXX methods are class static, but it's convenient to call them on
# an instance as we don't know Exposure's pixel type
# (and hence psfCandidate's exact type)
if psfCandidate.getWidth() == 0:
psfCandidate.setBorderWidth(self._borderWidth)
psfCandidate.setWidth(self._kernelSize +
2 * self._borderWidth)
psfCandidate.setHeight(self._kernelSize +
2 * self._borderWidth)
im = psfCandidate.getMaskedImage().getImage()
vmax = afwMath.makeStatistics(im, afwMath.MAX).getValue()
if not numpy.isfinite(vmax):
continue
psfCandidateList.append(psfCandidate)
if display and displayExposure:
ds9.dot("o",
source.getX() - mi.getX0(),
source.getY() - mi.getY0(),
size=4,
frame=frame,
ctype=ds9.CYAN)
except Exception as err:
logger.log(
pexLogging.Log.INFO,
"Failed to make a psfCandidate from source %d: %s" %
(source.getId(), err))
return psfCandidateList
def readFits(fileName, hdu=0, flags=0):
"""Read a ds9 region file, returning a ObjectMaskCatalog object
This method is called "readFits" to fool the butler. The corresponding mapper entry looks like
brightObjectMask: {
template: "deepCoadd/BrightObjectMasks/%(tract)d/BrightObjectMask-%(tract)d-%(patch)s-%(filter)s.reg"
python: "lsst.obs.subaru.objectMasks.ObjectMaskCatalog"
persistable: "PurePythonClass"
storage: "FitsCatalogStorage"
}
and this is the only way I know to get it to read a random file type, in this case a ds9 region file
This method expects to find files named as BrightObjectMask-%(tract)d-%(patch)s-%(filter)s.reg
The files should be structured as follows:
# Description of catalogue as a comment
# CATALOG: catalog-id-string
# TRACT: 0
# PATCH: 5,4
# FILTER: HSC-I
wcs; fk5
circle(RA, DEC, RADIUS) # ID: 1
The commented lines must be present, with the relevant fields such as tract patch and filter filled
in. The coordinate system must be listed as above. Each patch is specified as a circle, with an RA,
DEC, and Radius specified in decimal degrees. Only circles are supported as region definitions
currently.
"""
log = pexLog.getDefaultLog().createChildLog("ObjectMaskCatalog")
brightObjects = ObjectMaskCatalog()
checkedWcsIsFk5 = False
with open(fileName) as fd:
for lineNo, line in enumerate(fd.readlines(), 1):
line = line.rstrip()
if re.search(r"^\s*#", line):
#
# Parse any line of the form "# key : value" and put them into the metadata.
#
# The medatdata values must be defined as outlined in the above docstring
#
# The value of these three keys will be checked,
# so get them right!
#
mat = re.search(r"^\s*#\s*([a-zA-Z][a-zA-Z0-9_]+)\s*:\s*(.*)", line)
if mat:
key, value = mat.group(1).lower(), mat.group(2)
if key == "tract":
value = int(value)
brightObjects.table.getMetadata().set(key, value)
line = re.sub(r"^\s*#.*", "", line)
if not line:
continue
if re.search(r"^\s*wcs\s*;\s*fk5\s*$", line, re.IGNORECASE):
checkedWcsIsFk5 = True
continue
# This regular expression parses the regions file for each region to be masked,
# with the format as specified in the above docstring.
mat = re.search(r"^\s*circle(?:\s+|\s*\(\s*)"
"(\d+(?:\.\d*)([d]*))" "(?:\s+|\s*,\s*)"
"([+-]?\d+(?:\.\d*)([d]*))" "(?:\s+|\s*,\s*)"
"(\d+(?:\.\d*))([d'\"]*)" "(?:\s*|\s*\)\s*)"
"\s*#\s*ID:\s*(\d+)" "\s*$"
, line)
if mat:
ra, raUnit, dec, decUnit, radius, radiusUnit, _id = mat.groups()
_id = int(_id)
ra = convertToAngle(ra, raUnit, "ra", fileName, lineNo)
dec = convertToAngle(dec, decUnit, "dec", fileName, lineNo)
radius = convertToAngle(radius, radiusUnit, "radius", fileName, lineNo)
rec = brightObjects.addNew()
# N.b. rec["coord"] = Coord is not supported, so we have to use the setter
rec["id"] = _id
rec.setCoord(afwCoord.Fk5Coord(ra, dec))
rec["radius"] = radius
else:
log.warn("Unexpected line \"%s\" at %s:%d" % (line, fileName, lineNo))
if not checkedWcsIsFk5:
raise RuntimeError("Expected to see a line specifying an fk5 wcs")
# This makes the deep copy contiguous in memory so that a ColumnView can be exposed to Numpy
brightObjects._catalog = brightObjects._catalog.copy(True)
return brightObjects
# the GNU General Public License along with this program. If not,
# see <http://www.lsstcorp.org/LegalNotices/>.
#
"""
@brief Demonstrate the use of the logging framework from Python
"""
import lsst.pex.logging as log
from lsst.pex.logging import Rec, Prop, endr
if __name__ == "__main__":
# in any script or function where you want to log message, the first
# thing you should do is get create a Log from the DefaultLog
mylog = log.Log(log.getDefaultLog(), "myapp.myfunc")
# simple message can be sent with a function. The first argument is
# the level of message; choices are: DEBUG, INFO, WARN, FATAL.
mylog.log(mylog.INFO, "this is a simple message");
# If you want to send multiple messages and/or properties all in the
# same message, you can use the shift operator. Be sure to end the
# message with "endr"
Rec(mylog, mylog.WARN) << "ouch!" << Prop("NODE", 5) \
<< "something bad happened!" << endr
# Normally properties are not printed to the screen. To see these, we'll
# turn them on now.
#
# Outside the pipeline framework, the default logger is a ScreenLog.
def getBackground(image, backgroundConfig, nx=0, ny=0, algorithm=None):
"""!Estimate the background of an image (a thin layer on lsst.afw.math.makeBackground)
\param[in] image image whose background is to be computed
\param[in] backgroundConfig configuration (a BackgroundConfig)
\param[in] nx number of x bands; 0 for default
\param[in] ny number of y bands; 0 for default
\param[in] algorithm name of interpolation algorithm; see lsst.afw.math.BackgroundControl for details
"""
backgroundConfig.validate();
logger = pexLogging.getDefaultLog()
logger = pexLogging.Log(logger,"lsst.meas.algorithms.detection.getBackground")
if not nx:
nx = image.getWidth()//backgroundConfig.binSize + 1
if not ny:
ny = image.getHeight()//backgroundConfig.binSize + 1
displayBackground = lsstDebug.Info(__name__).displayBackground
if displayBackground:
import itertools
ds9.mtv(image, frame=1)
xPosts = numpy.rint(numpy.linspace(0, image.getWidth() + 1, num=nx, endpoint=True))
yPosts = numpy.rint(numpy.linspace(0, image.getHeight() + 1, num=ny, endpoint=True))
with ds9.Buffering():
for (xMin, xMax), (yMin, yMax) in itertools.product(zip(xPosts[:-1], xPosts[1:]),
zip(yPosts[:-1], yPosts[1:])):
ds9.line([(xMin, yMin), (xMin, yMax), (xMax, yMax), (xMax, yMin), (xMin, yMin)], frame=1)
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(reduce(lambda x, y: x | image.getMask().getPlaneBitMask(y),
backgroundConfig.ignoredPixelMask, 0x0))
sctrl.setNanSafe(backgroundConfig.isNanSafe)
pl = pexLogging.Debug("lsst.meas.algorithms.detection.getBackground")
pl.debug(3, "Ignoring mask planes: %s" % ", ".join(backgroundConfig.ignoredPixelMask))
if not algorithm:
algorithm = backgroundConfig.algorithm
bctrl = afwMath.BackgroundControl(algorithm, nx, ny,
backgroundConfig.undersampleStyle, sctrl,
backgroundConfig.statisticsProperty)
# TODO: The following check should really be done within afw/math. With the
# current code structure, it would need to be accounted for in the
# doGetImage() funtion in BackgroundMI.cc (which currently only checks
# against the interpoation settings which is not appropriate when
# useApprox=True) and/or the makeApproximate() function in
# afw/Approximate.cc.
# See ticket DM-2920: "Clean up code in afw for Approximate background
# estimation" (which includes a note to remove the following and the
# similar checks in pipe_tasks/matchBackgrounds.py once implemented)
#
# Check that config setting of approxOrder/binSize make sense
# (i.e. ngrid (= shortDimension/binSize) > approxOrderX) and perform
# appropriate undersampleStlye behavior.
if backgroundConfig.useApprox:
if not backgroundConfig.approxOrderY in (backgroundConfig.approxOrderX,-1):
raise ValueError("Error: approxOrderY not in (approxOrderX, -1)")
order = backgroundConfig.approxOrderX
minNumberGridPoints = backgroundConfig.approxOrderX + 1
if min(nx,ny) <= backgroundConfig.approxOrderX:
logger.warn("Too few points in grid to constrain fit: min(nx, ny) < approxOrder) "+
"[min(%d, %d) < %d]" % (nx, ny, backgroundConfig.approxOrderX))
if backgroundConfig.undersampleStyle == "THROW_EXCEPTION":
raise ValueError("Too few points in grid (%d, %d) for order (%d) and binsize (%d)" % (
nx, ny, backgroundConfig.approxOrderX, backgroundConfig.binSize))
elif backgroundConfig.undersampleStyle == "REDUCE_INTERP_ORDER":
if order < 1:
raise ValueError("Cannot reduce approxOrder below 0. " +
"Try using undersampleStyle = \"INCREASE_NXNYSAMPLE\" instead?")
order = min(nx, ny) - 1
logger.warn("Reducing approxOrder to %d" % order)
elif backgroundConfig.undersampleStyle == "INCREASE_NXNYSAMPLE":
newBinSize = min(image.getWidth(),image.getHeight())//(minNumberGridPoints-1)
if newBinSize < 1:
raise ValueError("Binsize must be greater than 0")
newNx = image.getWidth()//newBinSize + 1
newNy = image.getHeight()//newBinSize + 1
bctrl.setNxSample(newNx)
bctrl.setNySample(newNy)
logger.warn("Decreasing binSize from %d to %d for a grid of (%d, %d)" %
(backgroundConfig.binSize, newBinSize, newNx, newNy))
actrl = afwMath.ApproximateControl(afwMath.ApproximateControl.CHEBYSHEV, order, order,
backgroundConfig.weighting)
bctrl.setApproximateControl(actrl)
return afwMath.makeBackground(image, bctrl)
def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet,
kernel, background, testSources, config,
origVariance = False, nptsFull = 1e6, keepPlots = True, titleFs=14):
"""Plot diffim residuals for LOCAL and SPATIAL models"""
candidateResids = []
spatialResids = []
nonfitResids = []
for cell in kernelCellSet.getCellList():
for cand in cell.begin(True): # only look at good ones
# Be sure
if not (cand.getStatus() == afwMath.SpatialCellCandidate.GOOD):
continue
cand = diffimLib.cast_KernelCandidateF(cand)
diffim = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
orig = cand.getScienceMaskedImage()
ski = afwImage.ImageD(kernel.getDimensions())
kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
sk = afwMath.FixedKernel(ski)
sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
sdiffim = cand.getDifferenceImage(sk, sbg)
# trim edgs due to convolution
bbox = kernel.shrinkBBox(diffim.getBBox())
tdiffim = diffim.Factory(diffim, bbox)
torig = orig.Factory(orig, bbox)
tsdiffim = sdiffim.Factory(sdiffim, bbox)
if origVariance:
candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
np.sqrt(torig.getVariance().getArray())))
spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
np.sqrt(torig.getVariance().getArray())))
else:
candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
np.sqrt(tdiffim.getVariance().getArray())))
spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
np.sqrt(tsdiffim.getVariance().getArray())))
fullIm = diffExposure.getMaskedImage().getImage().getArray()
fullMask = diffExposure.getMaskedImage().getMask().getArray()
if origVariance:
fullVar = exposure.getMaskedImage().getVariance().getArray()
else:
fullVar = diffExposure.getMaskedImage().getVariance().getArray()
bitmaskBad = 0
bitmaskBad |= afwImage.MaskU.getPlaneBitMask('NO_DATA')
bitmaskBad |= afwImage.MaskU.getPlaneBitMask('SAT')
idx = np.where((fullMask & bitmaskBad) == 0)
stride = int(len(idx[0]) // nptsFull)
sidx = idx[0][::stride], idx[1][::stride]
allResids = fullIm[sidx] / np.sqrt(fullVar[sidx])
testFootprints = diffimTools.sourceToFootprintList(testSources, warpedTemplateExposure,
exposure, config, pexLog.getDefaultLog())
for fp in testFootprints:
subexp = diffExposure.Factory(diffExposure, fp["footprint"].getBBox())
subim = subexp.getMaskedImage().getImage()
if origVariance:
subvar = afwImage.ExposureF(exposure, fp["footprint"].getBBox()).getMaskedImage().getVariance()
else:
subvar = subexp.getMaskedImage().getVariance()
nonfitResids.append(np.ravel(subim.getArray() / np.sqrt(subvar.getArray())))
candidateResids = np.ravel(np.array(candidateResids))
spatialResids = np.ravel(np.array(spatialResids))
nonfitResids = np.ravel(np.array(nonfitResids))
try:
import pylab
from matplotlib.font_manager import FontProperties
except ImportError, e:
print "Unable to import pylab: %s" % e
return
def jobLog(job):
"""Add a job-specific log destination"""
if job is None or job == "None":
return
machine = os.uname()[1].split(".")[0]
pexLog.getDefaultLog().addDestination(job + ".%s.%d" % (machine, os.getpid()))
def __init__(self, config):
self.config = config
self.debugLog = pexLog.Debug("meas.algorithms.psfDeterminer")
self.warnLog = pexLog.Log(pexLog.getDefaultLog(), "meas.algorithms.psfDeterminer")
def __init__(self, config):
self.config = config
self.debugLog = pexLog.Debug("meas.algorithms.psfDeterminer")
self.warnLog = pexLog.Log(pexLog.getDefaultLog(),
"meas.algorithms.psfDeterminer")
def selectStars(self, exposure, catalog, matches=None):
"""Return a list of PSF candidates that represent likely stars
A list of PSF candidates may be used by a PSF fitter to construct a PSF.
@param[in] exposure: the exposure containing the sources
@param[in] catalog: a SourceCatalog containing sources that may be stars
@param[in] matches: astrometric matches; ignored by this star selector
@return psfCandidateList: a list of PSF candidates.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
plotMagSize = lsstDebug.Info(__name__).plotMagSize # display the magnitude-size relation
dumpData = lsstDebug.Info(__name__).dumpData # dump data to pickle file?
# create a log for my application
logger = pexLogging.Log(pexLogging.getDefaultLog(), "meas.algorithms.objectSizeStarSelector")
detector = exposure.getDetector()
distorter = None
xy0 = afwGeom.Point2D(0,0)
if not detector is None:
cPix = detector.getCenterPixel()
detSize = detector.getSize()
xy0.setX(cPix.getX() - int(0.5*detSize.getMm()[0]))
xy0.setY(cPix.getY() - int(0.5*detSize.getMm()[1]))
distorter = detector.getDistortion()
#
# Look at the distribution of stars in the magnitude-size plane
#
flux = catalog.get(self._sourceFluxField)
xx = numpy.empty(len(catalog))
xy = numpy.empty_like(xx)
yy = numpy.empty_like(xx)
for i, source in enumerate(catalog):
Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
if distorter:
xpix, ypix = source.getX() + xy0.getX(), source.getY() + xy0.getY()
p = afwGeom.Point2D(xpix, ypix)
m = distorter.undistort(p, geomEllip.Quadrupole(Ixx, Iyy, Ixy), detector)
Ixx, Ixy, Iyy = m.getIxx(), m.getIxy(), m.getIyy()
xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
width = numpy.sqrt(0.5*(xx + yy))
bad = reduce(lambda x, y: numpy.logical_or(x, catalog.get(y)), self._badFlags, False)
bad = numpy.logical_or(bad, flux < self._fluxMin)
bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
bad = numpy.logical_or(bad, width < self._widthMin)
bad = numpy.logical_or(bad, width > self._widthMax)
if self._fluxMax > 0:
bad = numpy.logical_or(bad, flux > self._fluxMax)
good = numpy.logical_not(bad)
if not numpy.any(good):
raise RuntimeError("No objects passed our cuts for consideration as psf stars")
mag = -2.5*numpy.log10(flux[good])
width = width[good]
#
# Look for the maximum in the size histogram, then search upwards for the minimum that separates
# the initial peak (of, we presume, stars) from the galaxies
#
if dumpData:
import os, cPickle as pickle
_ii = 0
while True:
pickleFile = os.path.expanduser(os.path.join("~", "widths-%d.pkl" % _ii))
if not os.path.exists(pickleFile):
break
_ii += 1
with open(pickleFile, "wb") as fd:
pickle.dump(mag, fd, -1)
pickle.dump(width, fd, -1)
centers, clusterId = _kcenters(width, nCluster=4, useMedian=True,
widthStdAllowed=self._widthStdAllowed)
if display and plotMagSize and pyplot:
fig = plot(mag, width, centers, clusterId, magType=self._sourceFluxField.split(".")[-1].title(),
marker="+", markersize=3, markeredgewidth=None, ltype=':', clear=True)
else:
fig = None
clusterId = _improveCluster(width, centers, clusterId,
nsigma = self._nSigmaClip, widthStdAllowed=self._widthStdAllowed)
if display and plotMagSize and pyplot:
plot(mag, width, centers, clusterId, marker="x", markersize=3, markeredgewidth=None, clear=False)
stellar = (clusterId == 0)
#
# We know enough to plot, if so requested
#
frame = 0
if fig:
if display and displayExposure:
ds9.mtv(exposure.getMaskedImage(), frame=frame, title="PSF candidates")
global eventHandler
eventHandler = EventHandler(fig.get_axes()[0], mag, width,
catalog.getX()[good], catalog.getY()[good], frames=[frame])
fig.show()
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
while True:
try:
reply = raw_input("continue? [c h(elp) q(uit) p(db)] ").strip()
except EOFError:
reply = None
if not reply:
reply = "c"
if reply:
if reply[0] == "h":
print """\
We cluster the points; red are the stellar candidates and the other colours are other clusters.
Points labelled + are rejects from the cluster (only for cluster 0).
At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in ds9.
"""
elif reply[0] == "p":
import pdb; pdb.set_trace()
elif reply[0] == 'q':
sys.exit(1)
else:
break
if display and displayExposure:
mi = exposure.getMaskedImage()
with ds9.Buffering():
for i, source in enumerate(catalog):
if good[i]:
ctype = ds9.GREEN # star candidate
else:
ctype = ds9.RED # not star
ds9.dot("+", source.getX() - mi.getX0(),
source.getY() - mi.getY0(), frame=frame, ctype=ctype)
#
# Time to use that stellar classification to generate psfCandidateList
#
with ds9.Buffering():
psfCandidateList = []
for isStellar, source in zip(stellar, [s for g, s in zip(good, catalog) if g]):
if not isStellar:
continue
try:
psfCandidate = algorithmsLib.makePsfCandidate(source, exposure)
# The setXXX methods are class static, but it's convenient to call them on
# an instance as we don't know Exposure's pixel type
# (and hence psfCandidate's exact type)
if psfCandidate.getWidth() == 0:
psfCandidate.setBorderWidth(self._borderWidth)
psfCandidate.setWidth(self._kernelSize + 2*self._borderWidth)
psfCandidate.setHeight(self._kernelSize + 2*self._borderWidth)
im = psfCandidate.getMaskedImage().getImage()
vmax = afwMath.makeStatistics(im, afwMath.MAX).getValue()
if not numpy.isfinite(vmax):
continue
psfCandidateList.append(psfCandidate)
if display and displayExposure:
ds9.dot("o", source.getX() - mi.getX0(), source.getY() - mi.getY0(),
size=4, frame=frame, ctype=ds9.CYAN)
except Exception as err:
logger.logdebug("Failed to make a psfCandidate from source %d: %s" % (source.getId(), err))
return psfCandidateList
def testAlgorithms(self):
"""Test that we can instantiate and use algorithms"""
config = measAlg.SourceMeasurementConfig()
config.algorithms.names = measAlg.AlgorithmRegistry.all.keys()
config.algorithms.names.discard(config.centroider.name)
config.doReplaceWithNoise = False
config.algorithms.names.discard("flux.peakLikelihood")
if False:
log = pexLog.getDefaultLog()
log.setThreshold(log.DEBUG)
schema = afwTable.SourceTable.makeMinimalSchema()
task = measAlg.SourceMeasurementTask(schema, config=config)
catalog = afwTable.SourceCatalog(schema)
source = catalog.addNew()
source.set("id", 12345)
size = 256
xStar, yStar = 65.432, 76.543
width = 3.21
x0, y0 = 12345, 54321
x, y = numpy.indices((size, size))
im = afwImage.MaskedImageF(afwGeom.ExtentI(size, size))
im.setXY0(afwGeom.Point2I(x0, y0))
im.getVariance().set(1.0)
arr = im.getImage().getArray()
arr[y,
x] = numpy.exp(-0.5 * ((x - xStar)**2 + (y - yStar)**2) / width**2)
psf = testLib.makeTestPsf(im)
exp = afwImage.makeExposure(im)
exp.setPsf(psf)
exp.setXY0(afwGeom.Point2I(x0, y0))
scale = 1.0e-5
wcs = afwImage.makeWcs(
afwCoord.Coord(0.0 * afwGeom.degrees, 0.0 * afwGeom.degrees),
afwGeom.Point2D(0.0, 0.0), scale, 0.0, 0.0, scale)
exp.setWcs(wcs)
point = afwGeom.Point2I(int(xStar + x0), int(yStar + y0))
bbox = im.getBBox()
bbox.shift(afwGeom.Extent2I(x0, y0))
foot = afwDetection.Footprint(point, width, bbox)
foot.addPeak(point.getX(), point.getY(), 1.0)
afwDetection.setMaskFromFootprint(exp.getMaskedImage().getMask(), foot,
1)
source.setFootprint(foot)
if display:
ds9.mtv(exp, frame=1)
task.run(exp, catalog)
for alg in config.algorithms:
flagName = alg + ".flags"
if False:
print(alg,
source.get(flagName) if flagName in schema else None,
source.get(alg) if alg in schema else None)
elif flagName in schema:
self.assertFalse(source.get(alg + ".flags"))
def selectStars(self, exposure, catalog, matches=None):
"""Return a list of PSF candidates that represent likely stars
A list of PSF candidates may be used by a PSF fitter to construct a PSF.
@param[in] exposure: the exposure containing the sources
@param[in] catalog: a SourceCatalog containing sources that may be stars
@param[in] matches: astrometric matches; ignored by this star selector
@return psfCandidateList: a list of PSF candidates.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = display and \
lsstDebug.Info(__name__).displayExposure # display the Exposure + spatialCells
plotFwhmHistogram = display and plt and \
lsstDebug.Info(__name__).plotFwhmHistogram # Plot histogram of FWHM
plotFlags = display and plt and \
lsstDebug.Info(__name__).plotFlags # Plot the sources coloured by their flags
plotRejection = display and plt and \
lsstDebug.Info(__name__).plotRejection # Plot why sources are rejected
# create a log for my application
logger = pexLogging.Log(pexLogging.getDefaultLog(), "meas.extensions.psfex.psfexStarSelector")
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
#
fluxName = self.config.fluxName
fluxErrName = self.config.fluxErrName
minFwhm = self.config.minFwhm
maxFwhm = self.config.maxFwhm
maxFwhmVariability = self.config.maxFwhmVariability
maxbad = self.config.maxbad
maxbadflag = self.config.maxbadflag
maxellip = self.config.maxellip
minsn = self.config.minsn
maxelong = (maxellip + 1.0)/(1.0 - maxellip) if maxellip < 1.0 else 100
# Unpack the catalogue
shape = catalog.getShapeDefinition()
ixx = catalog.get("%s.xx" % shape)
iyy = catalog.get("%s.yy" % shape)
fwhm = 2*np.sqrt(2*np.log(2))*np.sqrt(0.5*(ixx + iyy))
elong = 0.5*(ixx - iyy)/(ixx + iyy)
flux = catalog.get(fluxName)
fluxErr = catalog.get(fluxErrName)
sn = flux/np.where(fluxErr > 0, fluxErr, 1)
sn[fluxErr <= 0] = -psfex.psfex.cvar.BIG
flags = 0x0
for i, f in enumerate(self.config.badFlags):
flags = np.bitwise_or(flags, np.where(catalog.get(f), 1 << i, 0))
#
# Estimate the acceptable range of source widths
#
good = np.logical_and(sn > minsn, np.logical_not(flags))
good = np.logical_and(good, elong < maxelong)
good = np.logical_and(good, fwhm >= minFwhm)
good = np.logical_and(good, fwhm < maxFwhm)
fwhmMode, fwhmMin, fwhmMax = psfex.compute_fwhmrange(fwhm[good], maxFwhmVariability, minFwhm, maxFwhm,
plot=dict(fwhmHistogram=plotFwhmHistogram))
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
#
# Here's select_candidates
#
#---- Apply some selection over flags, fluxes...
bad = (flags != 0)
#set.setBadFlags(int(sum(bad)))
if plotRejection:
selectionVectors = []
selectionVectors.append((bad, "flags %d" % sum(bad)))
dbad = sn < minsn
#set.setBadSN(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "S/N %d" % sum(dbad)))
dbad = fwhm < fwhmMin
#set.setBadFrmin(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "fwhmMin %d" % sum(dbad)))
dbad = fwhm > fwhmMax
#set.setBadFrmax(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "fwhmMax %d" % sum(dbad)))
dbad = elong > maxelong
#set.setBadElong(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "elong %d" % sum(dbad)))
#-- ... and check the integrity of the sample
if maxbadflag:
nbad = np.array([(v <= -psfex.psfex.cvar.BIG).sum() for v in vignet])
dbad = nbad > maxbad
#set.setBadPix(int(sum(dbad)))
bad = np.logical_or(bad, dbad)
if plotRejection:
selectionVectors.append((dbad, "badpix %d" % sum(dbad)))
good = np.logical_not(bad)
#
# We know enough to plot, if so requested
#
frame = 0
if displayExposure:
mi = exposure.getMaskedImage()
ds9.mtv(mi, frame=frame, title="PSF candidates")
with ds9.Buffering():
for i, source in enumerate(catalog):
if good[i]:
ctype = ds9.GREEN # star candidate
else:
ctype = ds9.RED # not star
ds9.dot("+", source.getX() - mi.getX0(), source.getY() - mi.getY0(),
frame=frame, ctype=ctype)
if plotFlags or plotRejection:
imag = -2.5*np.log10(flux)
plt.clf()
alpha = 0.5
if plotFlags:
isSet = np.where(flags == 0x0)[0]
plt.plot(imag[isSet], fwhm[isSet], 'o', alpha=alpha, label="good")
for i, f in enumerate(self.config.badFlags):
mask = 1 << i
isSet = np.where(np.bitwise_and(flags, mask))[0]
if isSet.any():
if np.isfinite(imag[isSet] + fwhm[isSet]).any():
label = re.sub(r"\_flag", "",
re.sub(r"^base\_", "",
re.sub(r"^.*base\_PixelFlags\_flag\_", "", f)))
plt.plot(imag[isSet], fwhm[isSet], 'o', alpha=alpha, label=label)
else:
for bad, label in selectionVectors:
plt.plot(imag[bad], fwhm[bad], 'o', alpha=alpha, label=label)
plt.plot(imag[good], fwhm[good], 'o', color="black", label="selected")
[plt.axhline(_, color='red') for _ in [fwhmMin, fwhmMax]]
plt.xlim(np.median(imag[good]) + 5*np.array([-1, 1]))
plt.ylim(fwhm[np.where(np.isfinite(fwhm + imag))].min(), 2*fwhmMax)
plt.legend(loc=2)
plt.xlabel("Instrumental %s Magnitude" % fluxName.split(".")[-1].title())
plt.ylabel("fwhm")
title = "PSFEX Star Selection"
plt.title("%s %d selected" % (title, sum(good)))
if displayExposure:
global eventHandler
eventHandler = EventHandler(plt.axes(), imag, fwhm, catalog.getX(), catalog.getY(), frames=[frame])
if plotFlags or plotRejection:
while True:
try:
reply = raw_input("continue? [y[es] h(elp) p(db) q(uit)] ").strip()
except EOFError:
reply = "y"
if not reply:
reply = "y"
if reply[0] == "h":
print """\
At this prompt, you can continue with almost any key; 'p' enters pdb,
'q' returns to the shell, and
'h' prints this text
""",
if displayExposure:
print """
If you put the cursor on a point in the matplotlib scatter plot and hit 'p' you'll see it in ds9."""
elif reply[0] == "p":
import pdb; pdb.set_trace()
elif reply[0] == 'q':
sys.exit(1)
else:
break
#
# Time to use that stellar classification to generate psfCandidateList
#
with ds9.Buffering():
psfCandidateList = []
if True:
catalog = [s for s,g in zip(catalog, good) if g]
else:
catalog = catalog[good]
for source in catalog:
try:
psfCandidate = measAlg.makePsfCandidate(source, exposure)
# The setXXX methods are class static, but it's convenient to call them on
# an instance as we don't know Exposure's pixel type
# (and hence psfCandidate's exact type)
if psfCandidate.getWidth() == 0:
psfCandidate.setBorderWidth(self.config.borderWidth)
psfCandidate.setWidth(self.config.kernelSize + 2*self.config.borderWidth)
psfCandidate.setHeight(self.config.kernelSize + 2*self.config.borderWidth)
im = psfCandidate.getMaskedImage().getImage()
vmax = afwMath.makeStatistics(im, afwMath.MAX).getValue()
if not np.isfinite(vmax):
continue
psfCandidateList.append(psfCandidate)
if display and displayExposure:
ds9.dot("o", source.getX() - mi.getX0(), source.getY() - mi.getY0(),
size=4, frame=frame, ctype=ds9.CYAN)
except Exception as err:
logger.logdebug("Failed to make a psfCandidate from source %d: %s" % (source.getId(), err))
return psfCandidateList
def printProcessStats():
"""Print the process statistics to the log"""
from lsst.pex.logging import getDefaultLog
getDefaultLog().info("Process stats for %s: %s" % (NODE, processStats()))
def __init__(self, policy, repositoryDir,
root=None, registry=None, calibRoot=None, calibRegistry=None,
provided=None, outputRoot=None):
"""Initialize the CameraMapper.
@param policy (pexPolicy.Policy) Policy with per-camera defaults
already merged
@param repositoryDir (string) Policy repository for the subclassing
module (obtained with getRepositoryPath() on the
per-camera default dictionary)
@param root (string) Root directory for data
@param registry (string) Path to registry with data's metadata
@param calibRoot (string) Root directory for calibrations
@param calibRegistry (string) Path to registry with calibrations'
metadata
@param provided (list of strings) Keys provided by the mapper
@param outputRoot (string) Root directory for output data
"""
dafPersist.Mapper.__init__(self)
self.log = pexLog.Log(pexLog.getDefaultLog(), "CameraMapper")
# Dictionary
dictFile = pexPolicy.DefaultPolicyFile("daf_butlerUtils",
"MapperDictionary.paf", "policy")
dictPolicy = pexPolicy.Policy.createPolicy(dictFile,
dictFile.getRepositoryPath())
policy.mergeDefaults(dictPolicy)
# Levels
self.levels = dict()
if policy.exists("levels"):
levelsPolicy = policy.getPolicy("levels")
for key in levelsPolicy.names(True):
self.levels[key] = set(levelsPolicy.getStringArray(key))
self.defaultLevel = policy.getString("defaultLevel")
self.defaultSubLevels = dict()
if policy.exists("defaultSubLevels"):
defaultSubLevelsPolicy = policy.getPolicy("defaultSubLevels")
for key in defaultSubLevelsPolicy.names(True):
self.defaultSubLevels[key] = defaultSubLevelsPolicy.getString(key)
# Root directories
if root is None:
root = "."
root = dafPersist.LogicalLocation(root).locString()
if outputRoot is not None:
# Path manipulations are subject to race condition
if not os.path.exists(outputRoot):
try:
os.makedirs(outputRoot)
except OSError, e:
if not e.errno == errno.EEXIST:
raise
if not os.path.exists(outputRoot):
raise RuntimeError, "Unable to create output " \
"repository '%s'" % (outputRoot,)
if os.path.exists(root):
# Symlink existing input root to "_parent" in outputRoot.
src = os.path.abspath(root)
dst = os.path.join(outputRoot, "_parent")
if not os.path.exists(dst):
try:
os.symlink(src, dst)
except OSError:
pass
if os.path.exists(dst):
if os.path.realpath(dst) != os.path.realpath(src):
raise RuntimeError, "Output repository path " \
"'%s' already exists and differs from " \
"input repository path '%s'" % (dst, src)
else:
raise RuntimeError, "Unable to symlink from input " \
"repository path '%s' to output repository " \
"path '%s'" % (src, dst)
# We now use the outputRoot as the main root with access to the
# input via "_parent".
root = outputRoot
def __init__(self, root, log=None):
if log is None:
log = getDefaultLog()
self.log = log
self.root = root
self._butler = None
