def setUp(self):
self.qaTask = PipeQaTask()
self.testDatabase = "abecker_pipeQA_unittest" # lsstSim S21 schema
self.testVisit1 = "899551571" # z-band
self.testVisit2 = "899553091" # r-band
self.testFilt1 = "z"
self.testFilt2 = "r"
self.testRaft = "2,2"
self.testCcd = "1,1"
os.environ["WWW_ROOT"] = os.path.join(
eups.productDir("testing_pipeQA"), "tests")
self.wwwRoot = os.environ["WWW_ROOT"]
self.wwwRerun = "www_rerun"
os.environ["WWW_RERUN"] = self.wwwRerun
if os.path.isdir(self.wwwRerun):
shutil.rmtree(self.wwwRerun)
self.wwwPath = os.path.join(self.wwwRoot, self.wwwRerun)
# Test if database access is enabled; if not the tests will not be run
self.run = True
self.log = pexLog.Log(pexLog.Log.getDefaultLog(),
'lsst.testing.pipeQA.PipeQaTestCases',
pexLog.Log.INFO)
try:
dbid = DatabaseIdentity(self.testDatabase)
except pexExcept.LsstCppException, e:
self.log.warn("Unable to create database identity: %s" % e.message)
self.run = False
return
class MO(object):
"""Measure the sources on a frame"""
def __init__(self, display = False, rhs = None):
self.display = display
self.gas = None
if rhs:
try:
self.exposure = rhs.exposure
self.gas = rhs.gas
self.pixscale = rhs.pixscale
self.psf = rhs.psf
self.sourceList = rhs.sourceList
self.XY0 = rhs.XY0
try:
self.psfImage = rhs.psfImage
except AttributeError:
pass
except AttributeError, e:
raise RuntimeError, ("Unexpected rhs: %s (%s)" % (rhs, e))
self.display = display
# set up a log
self.scriptLog = pexLog.getDefaultLog()
self.scriptLog.setThreshold(pexLog.Log.WARN)
self.log = pexLog.Log(self.scriptLog, "measureSources")
def __init__(self):
# Set up defaults to send to deblender
# Always deblend as Psf
self.psfChisqCut1 = self.psfChisqCut2 = self.psfChisqCut2b = np.inf
self.log = pexLog.Log(pexLog.Log.getDefaultLog(),
'lsst.ip.diffim.DipoleDeblender',
pexLog.Log.INFO)
self.sigma2fwhm = 2. * np.sqrt(2. * np.log(2.))
def __init__(self, config):
"""
@param[in] config: instance of PolypixPsfDeterminerConfig
"""
self.config = config
# N.b. name of component is meas.algorithms.psfDeterminer so you can turn on psf debugging
# independent of which determiner is active
self.debugLog = pexLog.Debug("meas.algorithms.psfDeterminer")
self.warnLog = pexLog.Log(pexLog.getDefaultLog(), "meas.algorithms.psfDeterminer")
def makeCoadd(exposurePathList, policy):
"""Make a coadd using ChiSquaredStage
Inputs:
- exposurePathList: a list of paths to calibrated science exposures
- policy: policy to control the coaddition stage
"""
if len(exposurePathList) == 0:
print "No images specified; nothing to do"
return
# until PR 1069 is fixed one cannot actually use SimpleStageTester to process multiple files
# meanwhile just call the process method directly
# set up pipeline
stage = coaddPipe.ChiSquaredStage(policy)
# tester = pexHarness.simpleStageTester.SimpleStageTester(stage)
# tester.setDebugVerbosity(Verbosity)
tempLog = pexLog.Log()
parallelStage = coaddPipe.ChiSquaredStageParallel(policy, tempLog)
# process exposures
coadd = None
lastInd = len(exposurePathList) - 1
for expInd, exposurePath in enumerate(exposurePathList):
isFirst = (expInd == 0)
isLast = (expInd == lastInd)
print "Processing exposure %d of %d: %s" % (expInd + 1, lastInd + 1,
exposurePath)
try:
exposure = afwImage.ExposureF(filePath)
except Exception, e:
print "Skipping %s: %s" % (filePath, e)
continue
# set up the clipboard
clipboard = pexHarness.Clipboard.Clipboard()
inPolicy = policy.get("inputKeys")
clipboard.put(inPolicy.get("exposure"), exposure)
event = dafBase.PropertySet()
event.add("isLastExposure", isLast)
clipboard.put(inPolicy.get("event"), event)
# run the stage
# outClipboard = tester.runWorker(clipboard)
parallelStage.process(clipboard)
outClipboard = clipboard
if isLast:
# one could put this code after the loop, but then it is less robust against
# code changes (e.g. making multiple coadds), early exit, etc.
outPolicy = policy.get("outputKeys")
coadd = outClipboard.get(outPolicy.get("coadd"))
weightMap = outClipboard.get(outPolicy.get("weightMap"))
return (coadd, weightMap)
def setup(self):
self.log = pexLog.Log(self.log, self.__class__.__name__)
policyFile = pexPolicy.DefaultPolicyFile(self.packageName,
self.policyDictionaryName,
"policy")
defPolicy = pexPolicy.Policy.createPolicy(
policyFile, policyFile.getRepositoryPath(), True)
if self.policy is None:
self.policy = pexPolicy.Policy()
self.policy.mergeDefaults(defPolicy.getDictionary())
def importMatplotlib():
"""!Import matplotlib.pyplot when needed, warn if not available.
@return the imported module if available, else False.
"""
try:
import matplotlib.pyplot as plt
return plt
except ImportError as err:
log = pexLog.Log(pexLog.getDefaultLog(), 'lsst.ip.diffim.utils',
pexLog.INFO)
log.warn('Unable to import matplotlib: %s' % err)
return False
def __init__(self,
script,
importList=None,
command=None,
resourceList=None,
queue=None):
"""Initialisation
@param script Text of script to execute
@param importList List of imports; may be specified as a 2-tuple for 'import X as Y'
@param command Command to run to submit to queue ("qsub -V")
@param resourceList List of resources for PBS
@param queue Name of queue
"""
# Remove common indentation
lines = re.split("\n", script)
exemplar = None # First non-blank line
for line in lines:
if re.search("\S", line):
exemplar = line
break
if exemplar is None:
raise RuntimeError("Empty script provided.")
match = re.match("(\s+)", exemplar)
if match:
indent = match.group(0) # Indentation used
newLines = []
for line in lines:
if not re.search("\S", line):
continue
newLine = re.sub("^" + indent, "", line)
if newLine is None:
raise RuntimeError("Inconsistent indentation in script: " +
script)
newLines.append(newLine)
script = "\n".join(newLines)
self.script = script
self.importList = importList
self.command = command if command is not None else "qsub -V"
self.resourceList = resourceList
self.queue = queue
self.log = pexLog.Log(pexLog.getDefaultLog(), "PbsQueue")
return
def __init__(
self, # ReadWrite
mappers, # Mapper or mapper class to use
ccdKeys, # Data keywords required to specify a CCD
fileKeys=None, # Data keywords required to specify a file
config=None, # Configuration
):
"""Initialisation
@param mapper Data mapper (class or instance) for persistence
@param config Configuration (for instantiating mapper)
"""
# if we got a list, it contains [inMapper, outMapper]
if isinstance(mappers, list) and len(mappers) == 2:
inMapper, outMapper = mappers
# if we got a mapper, use it for both input and output
elif (isinstance(mappers, dafPersist.Mapper)
or issubclass(mappers, dafPersist.Mapper)):
inMapper, outMapper = mappers, mappers
# punt
else:
raise RuntimeError(
"'mapper' must be a dafPersist.Mapper (or derived from), or a list containing two of them (in and out)."
)
self.log = pexLog.Log(pexLog.getDefaultLog(), "ReadWrite")
self.inMapper = initMapper(inMapper, config, self.log, inMap=True)
self.ibf = dafPersist.ButlerFactory(mapper=self.inMapper)
self.inButler = self.ibf.create()
self.outMapper = initMapper(outMapper, config, self.log, inMap=False)
self.obf = dafPersist.ButlerFactory(mapper=self.outMapper)
self.outButler = self.obf.create()
self.ccdKeys = ccdKeys
if fileKeys is None:
fileKeys = list(ccdKeys)
if isinstance(fileKeys, basestring):
fileKeys = [fileKeys]
self.fileKeys = fileKeys
return
def __init__(self, config=None, log=None):
self.config = config
if log is None: log = pexLog.getDefaultLog()
self.log = pexLog.Log(log, self.__class__.__name__)
self._display = lsstDebug.Info(__name__).display
def selectStars(self, exposure, catalog, matches=None):
"""Return a list of PSF candidates that represent likely stars
A list of PSF candidates may be used by a PSF fitter to construct a PSF.
@param[in] exposure: the exposure containing the sources
@param[in] catalog: a SourceCatalog containing sources that may be stars
@param[in] matches: astrometric matches; ignored by this star selector
@return psfCandidateList: a list of PSF candidates.
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
displayExposure = lsstDebug.Info(
__name__).displayExposure # display the Exposure + spatialCells
plotMagSize = lsstDebug.Info(
__name__).plotMagSize # display the magnitude-size relation
dumpData = lsstDebug.Info(
__name__).dumpData # dump data to pickle file?
# create a log for my application
logger = pexLogging.Log(pexLogging.getDefaultLog(),
"meas.algorithms.objectSizeStarSelector")
detector = exposure.getDetector()
distorter = None
xy0 = afwGeom.Point2D(0, 0)
if not detector is None:
cPix = detector.getCenterPixel()
detSize = detector.getSize()
xy0.setX(cPix.getX() - int(0.5 * detSize.getMm()[0]))
xy0.setY(cPix.getY() - int(0.5 * detSize.getMm()[1]))
distorter = detector.getDistortion()
#
# Look at the distribution of stars in the magnitude-size plane
#
flux = catalog.get(self._sourceFluxField)
xx = numpy.empty(len(catalog))
xy = numpy.empty_like(xx)
yy = numpy.empty_like(xx)
for i, source in enumerate(catalog):
Ixx, Ixy, Iyy = source.getIxx(), source.getIxy(), source.getIyy()
if distorter:
xpix, ypix = source.getX() + xy0.getX(), source.getY(
) + xy0.getY()
p = afwGeom.Point2D(xpix, ypix)
m = distorter.undistort(p, geomEllip.Quadrupole(Ixx, Iyy, Ixy),
detector)
Ixx, Ixy, Iyy = m.getIxx(), m.getIxy(), m.getIyy()
xx[i], xy[i], yy[i] = Ixx, Ixy, Iyy
width = numpy.sqrt(xx + yy)
bad = reduce(lambda x, y: numpy.logical_or(x, catalog.get(y)),
self._badFlags, False)
bad = numpy.logical_or(bad, flux < self._fluxMin)
bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(width)))
bad = numpy.logical_or(bad, numpy.logical_not(numpy.isfinite(flux)))
bad = numpy.logical_or(bad, width < self._widthMin)
bad = numpy.logical_or(bad, width > self._widthMax)
if self._fluxMax > 0:
bad = numpy.logical_or(bad, flux > self._fluxMax)
good = numpy.logical_not(bad)
if not numpy.any(good):
raise RuntimeError(
"No objects passed our cuts for consideration as psf stars")
mag = -2.5 * numpy.log10(flux[good])
width = width[good]
#
# Look for the maximum in the size histogram, then search upwards for the minimum that separates
# the initial peak (of, we presume, stars) from the galaxies
#
if dumpData:
import os, cPickle as pickle
_ii = 0
while True:
pickleFile = os.path.expanduser(
os.path.join("~", "widths-%d.pkl" % _ii))
if not os.path.exists(pickleFile):
break
_ii += 1
with open(pickleFile, "wb") as fd:
pickle.dump(mag, fd, -1)
pickle.dump(width, fd, -1)
centers, clusterId = _kcenters(width, nCluster=4, useMedian=True)
if display and plotMagSize and pyplot:
fig = plot(mag,
width,
centers,
clusterId,
marker="+",
markersize=3,
markeredgewidth=None,
ltype=':',
clear=True)
else:
fig = None
clusterId = _improveCluster(width, centers, clusterId)
if display and plotMagSize and pyplot:
plot(mag,
width,
centers,
clusterId,
marker="x",
markersize=3,
markeredgewidth=None)
stellar = (clusterId == 0)
#
# We know enough to plot, if so requested
#
frame = 0
if fig:
if display and displayExposure:
ds9.mtv(exposure.getMaskedImage(),
frame=frame,
title="PSF candidates")
global eventHandler
eventHandler = EventHandler(fig.get_axes()[0],
mag,
width,
catalog.getX()[good],
catalog.getY()[good],
frames=[frame])
fig.show()
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
while True:
try:
reply = raw_input(
"continue? [c h(elp) q(uit) p(db)] ").strip()
except EOFError:
reply = "y"
if reply:
if reply[0] == "h":
print """\
We cluster the points; red are the stellar candidates and the other colours are other clusters.
Points labelled + are rejects from the cluster (only for cluster 0).
At this prompt, you can continue with almost any key; 'p' enters pdb, and 'h' prints this text
If displayExposure is true, you can put the cursor on a point and hit 'p' to see it in ds9.
"""
elif reply[0] == "p":
import pdb
pdb.set_trace()
elif reply[0] == 'q':
sys.exit(1)
else:
break
if display and displayExposure:
mi = exposure.getMaskedImage()
with ds9.Buffering():
for i, source in enumerate(catalog):
if good[i]:
ctype = ds9.GREEN # star candidate
else:
ctype = ds9.RED # not star
ds9.dot("+",
source.getX() - mi.getX0(),
source.getY() - mi.getY0(),
frame=frame,
ctype=ctype)
#
# Time to use that stellar classification to generate psfCandidateList
#
with ds9.Buffering():
psfCandidateList = []
for isStellar, source in zip(
stellar, [s for g, s in zip(good, catalog) if g]):
if not isStellar:
continue
try:
psfCandidate = algorithmsLib.makePsfCandidate(
source, exposure)
# The setXXX methods are class static, but it's convenient to call them on
# an instance as we don't know Exposure's pixel type
# (and hence psfCandidate's exact type)
if psfCandidate.getWidth() == 0:
psfCandidate.setBorderWidth(self._borderWidth)
psfCandidate.setWidth(self._kernelSize +
2 * self._borderWidth)
psfCandidate.setHeight(self._kernelSize +
2 * self._borderWidth)
im = psfCandidate.getMaskedImage().getImage()
vmax = afwMath.makeStatistics(im, afwMath.MAX).getValue()
if not numpy.isfinite(vmax):
continue
psfCandidateList.append(psfCandidate)
if display and displayExposure:
ds9.dot("o",
source.getX() - mi.getX0(),
source.getY() - mi.getY0(),
size=4,
frame=frame,
ctype=ds9.CYAN)
except Exception as err:
logger.log(
pexLogging.Log.INFO,
"Failed to make a psfCandidate from source %d: %s" %
(source.getId(), err))
return psfCandidateList
def makeCoadd(exposurePathList, warpExposurePolicy, psfMatchPolicy,
chiSquaredPolicy):
"""Make a coadd using psf-matching and chiSquaredStage
Inputs:
- exposurePathList: a list of paths to calibrated science exposures
- warpExposurePolicy: policy to control warping
- psfMatchPolicy: policy to control psf-matching
- chiSquaredPolicy: policy to control chi squared stage
"""
if len(exposurePathList) == 0:
print "No images specified; nothing to do"
return
# until PR 1069 is fixed one cannot actually use SimpleStageTester to process multiple files
# meanwhile just call the process method directly
# set up pipeline stages
# note: CoaddGenerationStage cannot be run with SimpleStageTester until PR 1069 is fixed.
warpExposureStage = coaddPipe.WarpExposureStage(warpExposurePolicy)
warpExposureTester = pexHarness.simpleStageTester.SimpleStageTester(
warpExposureStage)
warpExposureTester.setDebugVerbosity(Verbosity)
psfMatchStage = coaddPipe.PsfMatchStage(psfMatchPolicy)
psfMatchTester = pexHarness.simpleStageTester.SimpleStageTester(
psfMatchStage)
psfMatchTester.setDebugVerbosity(Verbosity)
# stage = coaddPipe.CoaddGenerationStage(chiSquaredPolicy)
# tester = pexHarness.simpleStageTester.SimpleStageTester(stage)
# tester.setDebugVerbosity(Verbosity)
tempLog = pexLog.Log()
chiSquaredStage = coaddPipe.ChiSquaredStageParallel(
chiSquaredPolicy, tempLog)
# process exposures
referenceExposure = None
lastInd = len(exposurePathList) - 1
for expInd, exposurePath in enumerate(exposurePathList):
isFirst = (expInd == 0)
isLast = (expInd == lastInd)
print "Processing exposure %d of %d: %s" % (expInd + 1, lastInd + 1,
exposurePath)
exposure = afwImage.ExposureF(exposurePath)
print "Subtract background"
subtractBackground(exposure.getMaskedImage())
clipboard = pexHarness.Clipboard.Clipboard()
event = dafBase.PropertySet()
event.add("isLastExposure", isLast)
clipboard.put(chiSquaredPolicy.get("inputKeys.event"), event)
# psf-match, if necessary
if not referenceExposure:
print "First exposure; simply add to coadd"
referenceExposure = exposure
clipboard.put(chiSquaredPolicy.get("inputKeys.exposure"), exposure)
else:
clipboard.put(warpExposurePolicy.get("inputKeys.exposure"),
exposure)
clipboard.put(
warpExposurePolicy.get("inputKeys.referenceExposure"),
referenceExposure)
print "Warp exposure"
warpExposureTester.runWorker(clipboard)
print "PSF-match exposure"
psfMatchTester.runWorker(clipboard)
psfMatchedExposure = clipboard.get(
psfMatchPolicy.get("outputKeys.psfMatchedExposure"))
clipboard.put(chiSquaredPolicy.get("inputKeys.exposure"),
psfMatchedExposure)
if SaveDebugImages:
print "Save debug images"
exposureName = os.path.basename(exposurePath)
warpedExposure = clipboard.get(
warpExposurePolicy.get("outputKeys.warpedExposure"))
warpedExposure.writeFits("warped_%s" % (exposureName, ))
psfMatchedExposure = clipboard.get(
psfMatchPolicy.get("outputKeys.psfMatchedExposure"))
psfMatchedExposure.writeFits("psfMatched_%s" %
(exposureName, ))
print "Add exposure to coadd"
# chiSquaredTester.runWorker(clipboard)
chiSquaredStage.process(clipboard)
if isLast:
# one could put this code after the loop, but then it is less robust against
# code changes (e.g. making multiple coadds), early exit, etc.
coadd = clipboard.get(chiSquaredPolicy.get("outputKeys.coadd"))
weightMap = clipboard.get(
chiSquaredPolicy.get("outputKeys.weightMap"))
return (coadd, weightMap)
raise RuntimeError("Unexpected error; last exposure never run")
# You should have received a copy of the LSST License Statement and
# the GNU General Public License along with this program. If not,
# see <http://www.lsstcorp.org/LegalNotices/>.
#
"""
@brief Demonstrate the use of the logging framework from Python
"""
import lsst.pex.logging as log
from lsst.pex.logging import Rec, Prop, endr
if __name__ == "__main__":
# in any script or function where you want to log message, the first
# thing you should do is get create a Log from the DefaultLog
mylog = log.Log(log.getDefaultLog(), "myapp.myfunc")
# simple message can be sent with a function. The first argument is
# the level of message; choices are: DEBUG, INFO, WARN, FATAL.
mylog.log(mylog.INFO, "this is a simple message")
# If you want to send multiple messages and/or properties all in the
# same message, you can use the shift operator. Be sure to end the
# message with "endr"
Rec(mylog, mylog.WARN) << "ouch!" << Prop("NODE", 5) \
<< "something bad happened!" << endr
# Normally properties are not printed to the screen. To see these, we'll
# turn them on now.
#
# Outside the pipeline framework, the default logger is a ScreenLog.
import lsst.pex.logging as pexLog
import lsst.daf.base as dafBase
import lsst.ctrl.events as ctrlEvents
from lsst.pex.harness import run
from lsst.ctrl.mospipe.MetadataStages import transformMetadata, validateMetadata
usage = """Usage: %prog [-dvqs] [-V lev] [-b host] [-t topic] FITSfile policyfile"""
desc = """Send an incoming visit event to instruct the alert production to process
a given FITS file. The data for the event is extracted from the FITS file.
The given policy file controls the transformation of the FITS metadata into
the event metadata where different input data collections (CFHT, simulated,
etc.) will require different policies. See the
$CTRL_DC3PIPE/pipeline/datatypePolicy directory for samples.
"""
logger = pexLog.Log(pexLog.Log.getDefaultLog(), "mospipe.eventFromFitsfile")
exposureCount = 0
VERB3 = run.verbosity2threshold("verb3", logger.INFO - 3)
def defineCmdLine(usage=usage, description=desc):
cl = optparse.OptionParser(usage=usage, description=description)
run.addAllVerbosityOptions(cl, "V", "verb")
cl.add_option("-b",
"--broker",
action="store",
dest="broker",
default="newfield.as.arizona.edu",
help="event broker host")
cl.add_option("-t",
"--topic",
fp, pathname, description = imp.find_module('eventFromFitsfile', [thisDir,])
try:
eventFromFitsfile = imp.load_module('eventFromFitsfile', fp, pathname,
description)
finally:
# Since we may exit via an exception, close fp explicitly.
if fp:
fp.close()
# Constants
EXP_TIME = 15.
SLEW_TIME = 5.
ROOT_EVENT_TOPIC = 'triggerImageprocEvent'
EVENT_BROKER = 'lsst4.ncsa.uiuc.edu'
logger = pexLog.Log(pexLog.Log.getDefaultLog(),
"dc3pipe.eventFromFitsfileList")
visitCount = 0
def EventFromInputFileList(inputfile,
datatypePolicy,
expTime=EXP_TIME,
slewTime=SLEW_TIME,
maxvisits=-1,
rootTopicName=ROOT_EVENT_TOPIC,
hostName=EVENT_BROKER,
metadataPolicy=None):
"""
Generate events for the IPSD (and MOPS) pipeline by reading a list of visit
directories and extracting the relevant information from the FITS files
therein.
def astrometry(self,
exposure,
sources,
distSources,
distortion=None,
llc=(0, 0),
size=None):
"""Solve astrometry to produce WCS
@param exposure Exposure to process
@param sources Sources as measured (actual) positions
@param distSources Sources with undistorted (ideal) positions
@param distortion Distortion model
@param llc Lower left corner (minimum x,y)
@param size Size of exposure
@return Star matches, match metadata
"""
assert exposure, "No exposure provided"
assert distSources, "No sources provided"
self.log.log(self.log.INFO, "Solving astrometry")
if size is None:
size = (exposure.getWidth(), exposure.getHeight())
try:
menu = self.config['filters']
filterName = menu[exposure.getFilter().getName()]
if isinstance(filterName, pipConfig.Config):
filterName = filterName['primary']
self.log.log(self.log.INFO,
"Using catalog filter: %s" % filterName)
except:
self.log.log(
self.log.WARN,
"Unable to determine catalog filter from lookup table using %s"
% exposure.getFilter().getName())
filterName = None
if distortion is not None:
# Removed distortion, so use low order
oldOrder = self.config['astrometry']['sipOrder']
self.config['astrometry']['sipOrder'] = 2
log = pexLog.Log(self.log, "astrometry")
astrom = measAst.determineWcs(self.config['astrometry'].getPolicy(),
exposure,
distSources,
log=log,
forceImageSize=size,
filterName=filterName)
if distortion is not None:
self.config['astrometry']['sipOrder'] = oldOrder
if astrom is None:
raise RuntimeError("Unable to solve astrometry for %s",
exposure.getDetector().getId())
wcs = astrom.getWcs()
matches = astrom.getMatches()
matchMeta = astrom.getMatchMetadata()
if matches is None or len(matches) == 0:
raise RuntimeError("No astrometric matches for %s",
exposure.getDetector().getId())
self.log.log(self.log.INFO, "%d astrometric matches for %s" % \
(len(matches), exposure.getDetector().getId()))
exposure.setWcs(wcs)
# Apply WCS to sources
for index, source in enumerate(sources):
distSource = distSources[index]
sky = wcs.pixelToSky(distSource.getXAstrom() - llc[0],
distSource.getYAstrom() - llc[1])
source.setRaDec(sky)
#point = afwGeom.Point2D(distSource.getXAstrom() - llc[0], distSource.getYAstrom() - llc[1])
# in square degrees
#areas.append(wcs.pixArea(point))
self.display('astrometry',
exposure=exposure,
sources=sources,
matches=matches)
return matches, matchMeta
eventFromFitsfile = imp.load_module('eventFromFitsfile', fp, pathname,
description)
finally:
# Since we may exit via an exception, close fp explicitly.
if fp:
fp.close()
# Constants
EXP_TIME = 15.
SLEW_TIME = 5.
ROOT_DIR = '/lsst/images/repository/raw'
SUBSETS = ['ALL', 'D1', 'D2', 'D3', 'D4']
ROOT_EVENT_TOPIC = 'triggerImageprocEvent'
EVENT_BROKER = 'lsst4.ncsa.uiuc.edu'
logger = pexLog.Log(pexLog.Log.getDefaultLog(),
"dc3pipe.eventGeneratorForCFHT")
visitCount = 0
def EventFromInputSubsets(subsets,
datatypePolicy,
expTime=EXP_TIME,
slewTime=SLEW_TIME,
maxvisits=-1,
rootTopicName=ROOT_EVENT_TOPIC,
hostName=EVENT_BROKER,
metadataPolicy=None):
"""
Generate events for the IPSD (and MOPS) pipeline by extracting the relevant
information from the FITS files in the standard DC3 CFHT subdirectories.
def astrometry(self,
exposure,
sources,
distSources,
distortion=None,
llc=(0, 0),
size=None):
"""Solve astrometry to produce WCS
@param exposure Exposure to process
@param sources Sources as measured (actual) positions
@param distSources Sources with undistorted (actual) positions
@param distortion Distortion model
@param llc Lower left corner (minimum x,y)
@param size Size of exposure
@return Star matches, match metadata
"""
assert exposure, "No exposure provided"
assert distSources, "No sources provided"
self.log.log(self.log.INFO, "Solving astrometry")
try:
import hsc.meas.astrom as hscAst
except ImportError:
hscAst = None
wcs = exposure.getWcs()
if wcs is None or hscAst is None:
self.log.log(
self.log.WARN,
"Unable to use hsc.meas.astrom; reverting to lsst.meas.astrom")
return super(CalibrateHsc, self).astrometry(exposure,
sources,
distSources,
distortion=distortion,
llc=llc,
size=size)
if size is None:
size = (exposure.getWidth(), exposure.getHeight())
try:
menu = self.config['filters']
filterName = menu[exposure.getFilter().getName()]
if isinstance(filterName, pipConfig.Config):
filterName = filterName['primary']
self.log.log(self.log.INFO,
"Using catalog filter: %s" % filterName)
except:
self.log.log(
self.log.WARN,
"Unable to determine catalog filter from lookup table using %s"
% exposure.getFilter().getName())
filterName = None
if distortion is not None:
# Removed distortion, so use low order
oldOrder = self.config['astrometry']['sipOrder']
self.config['astrometry']['sipOrder'] = 2
log = pexLog.Log(self.log, "astrometry")
wcs.shiftReferencePixel(-llc[0], -llc[1])
try:
astrom = hscAst.determineWcs(self.config['astrometry'].getPolicy(),
exposure,
distSources,
log=log,
forceImageSize=size,
filterName=filterName)
wcs.shiftReferencePixel(llc[0], llc[1])
if astrom is None:
raise RuntimeError(
"hsc.meas.astrom failed to determine the WCS")
except Exception, e:
self.log.log(
self.log.WARN,
"hsc.meas.astrom failed (%s); trying lsst.meas.astrom" % e)
astrom = measAstrom.determineWcs(
self.config['astrometry'].getPolicy(),
exposure,
distSources,
log=log,
forceImageSize=size,
filterName=filterName)
def __init__(
self,
bins=4,
doBackground=True,
scaleDetected=10.0,
sigmaSmooth=1.0,
thetaTolerance=0.15,
luminosityLimit=0.02,
centerLimit=1.2,
eRange=0.08,
bLimit=1.4,
skewLimit=10.0,
kernelSigma=7,
kernelWidth=11,
growKernel=1.4,
houghBins=200,
houghThresh=40,
maxTrailWidth=2.1,
maskAndBits=(),
log=None,
verbose=False,
):
"""Construct SatelliteFinder
@param bins Binning to use (improves speed, but becomes unreliable above 4)
@param doBackground Subtract median-ring filter background
@param scaleDetected Scale pixels with detected flag by this amount.
@param growKernel Repeat with a kernel larger by this fraction (no repeat if 1.0)
@param sigmaSmooth Do a Gaussian smooth with this sigma (binned pixels)
@param thetaTolerance Max theta difference for thetaAlignment() routine.
@param luminosityLimit Min flux to accept [units of Std.Dev].
@param centerLimit Max error in 1st moment (centroid) to accept [pixels].
@param eRange Max error in e=1-b/a above and below the calib trail value.
@param bLimit Max error in trail width [pixels].
@param skewLimit Max error in 3rd moment (skewness) to accept [pixels^3]
@param kernelSigma Gaussian sigma to taper the kernel [pixels]
@param kernelWidth Width of kernel in pixels.
@param houghBins Number of bins in r,theta space (total = bins x bins)
@param houghThresh Count level in Hough bin to consider a detection.
@param maxTrailWidth Discard trail detections wider than this.
@param maskAndBits Only allow pixels with these bits to be masked (typ. DETECTED, if any)
@param log A log object.
@param verbose Be chatty.
"""
self.bins = bins
self.doBackground = doBackground
self.scaleDetected = scaleDetected
self.sigmaSmooth = sigmaSmooth
self.thetaTolerance = thetaTolerance
self.kernelSigma = kernelSigma
self.kernelWidth = kernelWidth
self.growKernel = 1.4
self.centerLimit = centerLimit
self.eRange = eRange
self.houghThresh = houghThresh
self.houghBins = houghBins
self.luminosityLimit = luminosityLimit
self.skewLimit = skewLimit
self.bLimit = bLimit
self.maxTrailWidth = maxTrailWidth
self.maskAndBits = maskAndBits
if log is None:
logLevel = pexLog.Log.INFO
if verbose:
logLevel = pexLog.Log.DEBUG
log = pexLog.Log(pexLog.Log.getDefaultLog(), 'satelliteFinder',
logLevel)
self.log = log
self.debugInfo = {}
import numpy as np
import lsst.afw.detection as afwDet
import lsst.afw.geom as afwGeom
import lsst.afw.image as afwImage
import lsst.pex.logging as pexLogging
from lsst.meas.deblender.baseline import *
from lsst.meas.deblender import BaselineUtilsF as butils
import lsst.meas.algorithms as measAlg
root = pexLogging.Log.getDefaultLog()
root.setThreshold(pexLogging.Log.DEBUG)
# Quiet some of the more chatty loggers
pexLogging.Log(root, 'lsst.meas.deblender.symmetrizeFootprint',
pexLogging.Log.INFO)
pexLogging.Log(root, 'lsst.meas.deblender.symmetricFootprint',
pexLogging.Log.INFO)
pexLogging.Log(root, 'lsst.meas.deblender.getSignificantEdgePixels',
pexLogging.Log.INFO)
pexLogging.Log(root, 'afw.Mask',
pexLogging.Log.INFO)
def imExt(img):
bbox = img.getBBox(afwImage.PARENT)
return [bbox.getMinX(), bbox.getMaxX(),
bbox.getMinY(), bbox.getMaxY()]
def doubleGaussianPsf(W, H, fwhm1, fwhm2, a2):
return measAlg.DoubleGaussianPsf(W, H, fwhm1, fwhm2, a2)
def __init__(self, config):
self.config = config
self.debugLog = pexLog.Debug("meas.algorithms.psfDeterminer")
self.warnLog = pexLog.Log(pexLog.getDefaultLog(),
"meas.algorithms.psfDeterminer")
# You should have received a copy of the LSST License Statement and
# the GNU General Public License along with this program. If not,
# see <http://www.lsstcorp.org/LegalNotices/>.
#
import re
import lsst.afw.image as afwImage
import lsst.daf.base as dafBase
import lsst.pex.logging as pexLog
import lsst.utils as lsstutils
from lsst.pex.harness.Stage import Stage
propertySetTypeInfos = {}
logger = pexLog.Log(pexLog.Log.getDefaultLog(), "mospipe.MetadataStages.py")
def setTypeInfos():
global propertySetTypeInfos
p = dafBase.PropertySet()
p.set("str", "bar")
propertySetTypeInfos["string"] = p.typeOf("str")
p.set("int", 3)
propertySetTypeInfos["int"] = p.typeOf("int")
p.set("float", 3.1)
propertySetTypeInfos["float"] = p.typeOf("float")
p.set("bool", True)
propertySetTypeInfos["bool"] = p.typeOf("bool")
