def processCCDs(image):
from lsst.pipe.tasks.calibrate import CalibrateTask, CalibrateConfig
from lsst.pipe.tasks.characterizeImage import CharacterizeImageTask
calibRes = None
# init tasks
charImage = CharacterizeImageTask()
calibrateConfig = CalibrateConfig(doPhotoCal=False,
doAstrometry=False,
doDeblend=False)
calibrateTask = CalibrateTask(config=calibrateConfig)
try:
# characterize image
charRes = charImage.characterize(image,
exposureIdInfo=None,
background=None)
# calibrate image
calibRes = calibrateTask.calibrate(charRes.exposure,
exposureIdInfo=None,
background=charRes.background,
icSourceCat=None)
except Exception as e:
print "failed to calibrate the image"
print str(e)
return calibRes
def run(display=False):
#
# Create the tasks
#
charImageConfig = CharacterizeImageTask.ConfigClass()
charImageTask = CharacterizeImageTask(config=charImageConfig)
config = CalibrateTask.ConfigClass()
config.astrometry.retarget(MyAstrometryTask)
calibrateTask = CalibrateTask(config=config)
# load the data
# Exposure ID and the number of bits required for exposure IDs are usually obtained from a data repo,
# but here we pick reasonable values (there are 64 bits to share between exposure IDs and source IDs).
exposure = loadData()
exposureIdInfo = ExposureIdInfo(expId=1, expBits=5)
# characterize the exposure to repair cosmic rays and fit a PSF model
# display now because CalibrateTask modifies the exposure in place
charRes = charImageTask.characterize(exposure=exposure,
exposureIdInfo=exposureIdInfo)
if display:
displayFunc(charRes.exposure, charRes.sourceCat, frame=1)
# calibrate the exposure
calRes = calibrateTask.calibrate(exposure=charRes.exposure,
exposureIdInfo=exposureIdInfo)
if display:
displayFunc(calRes.exposure, calRes.sourceCat, frame=2)
def run(display=False):
#
# Create the task using a butler constructed using the obs_test repo
#
butler = dafPersistence.Butler(os.path.join(lsst.utils.getPackageDir("obs_test"), "data", "input"))
charImageConfig = CharacterizeImageTask.ConfigClass()
charImageTask = CharacterizeImageTask(butler, config=charImageConfig)
config = CalibrateTask.ConfigClass()
config.astrometry.retarget(MyAstrometryTask)
calibrateTask = CalibrateTask(butler, config=config)
# load the data
# Exposure ID and the number of bits required for exposure IDs are usually obtained from a data repo,
# but here we pick reasonable values (there are 64 bits to share between exposure IDs and source IDs).
exposure = loadData()
exposureIdInfo = ExposureIdInfo(expId=1, expBits=5)
# characterize the exposure to repair cosmic rays and fit a PSF model
# display now because CalibrateTask modifies the exposure in place
charRes = charImageTask.characterize(exposure=exposure, exposureIdInfo=exposureIdInfo)
if display:
displayFunc(charRes.exposure, charRes.sourceCat, frame=1)
# calibrate the exposure
calRes = calibrateTask.calibrate(exposure=charRes.exposure, exposureIdInfo=exposureIdInfo)
if display:
displayFunc(calRes.exposure, calRes.sourceCat, frame=2)
def run(display=False):
#
# Create the tasks
#
charImageConfig = CharacterizeImageTask.ConfigClass()
charImageTask = CharacterizeImageTask(config=charImageConfig)
config = CalibrateTask.ConfigClass()
config.astrometry.retarget(MyAstrometryTask)
calibrateTask = CalibrateTask(config=config)
# load the data
# Exposure ID and the number of bits required for exposure IDs are usually obtained from a data repo,
# but here we pick reasonable values (there are 64 bits to share between exposure IDs and source IDs).
exposure = loadData()
exposureIdInfo = ExposureIdInfo(expId=1, expBits=5)
# characterize the exposure to repair cosmic rays and fit a PSF model
# display now because CalibrateTask modifies the exposure in place
charRes = charImageTask.characterize(exposure=exposure, exposureIdInfo=exposureIdInfo)
if display:
displayFunc(charRes.exposure, charRes.sourceCat, frame=1)
# calibrate the exposure
calRes = calibrateTask.calibrate(exposure=charRes.exposure, exposureIdInfo=exposureIdInfo)
if display:
displayFunc(calRes.exposure, calRes.sourceCat, frame=2)
def run(display=False):
#
# Create the task
#
config = CalibrateTask.ConfigClass()
config.initialPsf.pixelScale = 0.185 # arcsec per pixel
config.initialPsf.fwhm = 1.0
config.astrometry.retarget(MyAstrometryTask)
calibrateTask = CalibrateTask(config=config)
#
# Process the data
#
exposure = loadData(config.initialPsf.pixelScale)
result = calibrateTask.run(exposure)
exposure0, exposure = exposure, result.exposure
sources = result.sources
if display: # display on ds9 (see also --debug argparse option)
frame = 1
ds9.mtv(exposure, frame=frame)
with ds9.Buffering():
for s in sources:
xy = s.getCentroid()
ds9.dot('+', *xy, ctype=ds9.CYAN if s.get("flags_negative") else ds9.GREEN, frame=frame)
def write_output_data(self, dataRef, calRes):
"""Write output data thru Butler based on config"""
if self.config.doWrite:
CalibrateTask.writeOutputs(
CalibrateTask(config=self.config, log=self.log, icSourceSchema=self.icSourceSchema),
dataRef=dataRef,
exposure=calRes.exposure,
background=calRes.background,
sourceCat=calRes.sourceCat,
astromMatches=calRes.astromMatches,
matchMeta=self.matchMeta,
)
def _checkDoRefcats(self, doAstrometry, doPhotoCal, ids):
"""Test whether run is called with the correct arguments.
In the case of `CalibrateTask`, the inputs should not depend on the
task configuration.
Parameters
----------
doAstrometry, doPhotoCal : `bool`
Values of the config flags of the same name.
ids : `dict` [`str`]
A mapping from the input dataset type to the data ID of the
dataset to process.
"""
config = CalibrateConfig()
config.doWriteMatches = False # no real output to write
config.doAstrometry = doAstrometry
config.doPhotoCal = doPhotoCal
config.connections.photoRefCat = "cal_ref_cat"
config.connections.astromRefCat = "cal_ref_cat"
task = CalibrateTask(config=config)
quantumId = ids["exposure"]
quantum = testUtils.makeQuantum(task, self.butler, quantumId, ids)
run = testUtils.runTestQuantum(task, self.butler, quantum)
run.assert_called_once()
self.assertEqual(run.call_args[0], ())
# Some arguments unprintable because we don't have a full environment
# So just check which ones were passed in
self.assertEqual(
run.call_args[1].keys(),
{"exposure", "exposureIdInfo", "background", "icSourceCat"})
def testIsPrimaryFlag(self):
"""Tests detect_isPrimary column gets added when run, and that sources
labelled as detect_isPrimary are not sky sources and have no children.
"""
charImConfig = CharacterizeImageConfig()
charImTask = CharacterizeImageTask(config=charImConfig)
charImResults = charImTask.run(self.exposure)
calibConfig = CalibrateConfig()
calibConfig.doAstrometry = False
calibConfig.doPhotoCal = False
calibTask = CalibrateTask(config=calibConfig)
calibResults = calibTask.run(charImResults.exposure)
outputCat = calibResults.outputCat
self.assertTrue("detect_isPrimary" in outputCat.schema.getNames())
# make sure all sky sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])),
0)
# make sure all parent sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"])
& (outputCat["deblend_nChild"] > 0)), 0)
def testIsSinglePrimaryFlag(self):
"""Tests detect_isPrimary column gets added when run, and that sources
labelled as detect_isPrimary are not sky sources and have no children.
"""
calibConfig = CalibrateConfig()
calibConfig.doAstrometry = False
calibConfig.doPhotoCal = False
calibTask = CalibrateTask(config=calibConfig)
calibResults = calibTask.run(self.charImResults.exposure)
outputCat = calibResults.outputCat
self.assertTrue("detect_isPrimary" in outputCat.schema.getNames())
# make sure all sky sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"]) & (outputCat["sky_source"])),
0)
# make sure all parent sources are flagged as not primary
self.assertEqual(
sum((outputCat["detect_isPrimary"])
& (outputCat["deblend_nChild"] > 0)), 0)
with self.assertRaises(KeyError):
outputCat.getSchema().find("detect_isDelendedModelPrimary")
def _checkSkySourceColumnExistence(self, doSkySources):
"""Implements sky_source column checking.
Parameters
----------
doSkySource : `bool`
Value of the config flag determining whether to insert sky sources.
"""
charImConfig = CharacterizeImageConfig()
charImConfig.measurePsf.psfDeterminer = 'piff'
charImConfig.measurePsf.psfDeterminer['piff'].spatialOrder = 0
charImTask = CharacterizeImageTask(config=charImConfig)
charImResults = charImTask.run(self.exposure)
calibConfig = CalibrateConfig()
calibConfig.doAstrometry = False
calibConfig.doPhotoCal = False
calibConfig.doSkySources = doSkySources
calibTask = CalibrateTask(config=calibConfig)
calibResults = calibTask.run(charImResults.exposure)
if doSkySources:
self.assertTrue('sky_source' in calibResults.outputCat.schema.getNames())
else:
self.assertFalse('sky_source' in calibResults.outputCat.schema.getNames())
def execute(self, dataRef):
"""!Characterize a science image
@param dataRef: butler data reference
@return a pipeBase Struct containing the results
"""
self.log.info("Performing Super Calibrate on sensor data ID %s" % (dataRef.dataId,))
self.log.info("Reading input data using dataRef")
inputData = self.read_input_data(dataRef)
self.log.info("Running operations. The run() method should not take anything Butler")
if self.config.doWrite and self.config.doAstrometry:
self.matchMeta = createMatchMetadata(inputData.getDict()['exposure'], border=self.pixelMargin)
else:
self.matchMeta = None
result = CalibrateTask.calibrate(CalibrateTask(config=self.config, log=self.log, icSourceSchema=self.icSourceSchema),
**inputData.getDict())
self.log.info("Writing output data using dataRef")
self.write_output_data(dataRef, result)
return result
def run(config, inputFiles, weightFiles=None, varianceFiles=None,
returnCalibSources=False, display=False, verbose=False):
#
# Create the tasks
#
schema = afwTable.SourceTable.makeMinimalSchema()
algMetadata = dafBase.PropertyList()
calibrateTask = CalibrateTask(config=config.calibrate)
sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema)
if config.doDeblend:
if SourceDeblendTask:
sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema)
else:
print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False"
config.doDeblend = False
sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement,
schema=schema, algMetadata=algMetadata)
exposureDict = {}; calibSourcesDict = {}; sourcesDict = {}
for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles):
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# read the data
#
if verbose:
print "Reading %s" % inputFile
exposure = makeExposure(inputFile, weightFile, varianceFile,
config.badPixelValue, config.variance, verbose)
#
# process the data
#
calibSources = None # sources used to calibrate the frame (photom, astrom, psf)
if config.doCalibrate:
result = calibrateTask.run(exposure)
exposure, sources = result.exposure, result.sources
if returnCalibSources:
calibSources = sources
else:
if not exposure.getPsf():
calibrateTask.installInitialPsf(exposure)
if config.edgeRolloff.applyModel:
if verbose:
print "Adding edge rolloff distortion to the exposure WCS"
addEdgeRolloffDistortion(exposure, config.edgeRolloff)
exposureDict[inputFile] = exposure
calibSourcesDict[inputFile] = calibSources
result = sourceDetectionTask.run(tab, exposure)
sources = result.sources
sourcesDict[inputFile] = sources
if config.doDeblend:
sourceDeblendTask.run(exposure, sources, exposure.getPsf())
sourceMeasurementTask.measure(exposure, sources)
if verbose:
print "Detected %d objects" % len(sources)
if display: # display on ds9 (see also --debug argparse option)
if algMetadata.exists("base_CircularApertureFlux_radii"):
radii = algMetadata.get("base_CircularApertureFlux_radii")
else:
radii = None
frame = 1
ds9.mtv(exposure, title=os.path.split(inputFile)[1], frame=frame)
with ds9.Buffering():
for s in sources:
xy = s.getCentroid()
ds9.dot('+', *xy, ctype=ds9.CYAN if s.get("flags_negative") else ds9.GREEN, frame=frame)
ds9.dot(s.getShape(), *xy, ctype=ds9.RED, frame=frame)
if radii:
for radius in radii:
ds9.dot('o', *xy, size=radius, ctype=ds9.YELLOW, frame=frame)
return exposureDict, calibSourcesDict, sourcesDict
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp(
) if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(config=getObsTestConfig(IsrTask), name="isr2")
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask), name="charImage2")
calibrateTask = CalibrateTask(config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema)
try:
dataId = dict(visit=1)
dataIdStrList = [
"%s=%s" % (key, val) for key, val in dataId.items()
]
isrResult1 = IsrTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
exposureIdInfo = inputButler.get("expIdInfo", dataId=dataId)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get(
"postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage())
icResult1 = CharacterizeImageTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure,
exposureIdInfo=exposureIdInfo)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get(
"icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId,
immediate=True),
icResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground",
dataId,
immediate=True),
icResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background)
calResult1 = CalibrateTask.parseAndRun(
args=[
InputDir, "--output", outPath, "--clobber-config",
"--doraise", "--id"
] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat,
exposureIdInfo=exposureIdInfo,
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get(
"calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage())
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat)
self.assertCatalogsEqual(calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent"))
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground",
dataId,
immediate=True),
calResult1.resultList[0].result.background)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" %
(OutputName, ))
####### Setup LSST begins
########################################
visit=289697
ccdnum = 1
HOME_PATH = os.path.expanduser("~")
butler = dafPersist.Butler(HOME_PATH + '/lsst_data/raw')
exposure = butler.get("instcal", visit=visit, ccdnum=ccdnum, filter='g', immediate=True)
mask = exposure.getMaskedImage().getMask().getArray() #TODO: move this matrix to SciDB
variance = exposure.getMaskedImage().getVariance().getArray() #TODO: move this matrix to SciDB
butler = dafPersist.Butler(HOME_PATH + '/lsst_data/raw')
filename = HOME_PATH + "/lsst_data/raw/crblasted0289697/instcal0289697.1.fits"
fitsHeader = afwImage.readMetadata(filename)
wcs = afwImage.makeWcs(fitsHeader)
charImage = CharacterizeImageTask()
calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False)
calibrateTask = CalibrateTask(config=calibrateConfig)
newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig(projection='STG',
decList=[-4.9325280994132905],
patchInnerDimensions=[2000, 2000],
radiusList=[4.488775723429071],
pixelScale=0.333, rotation=0.0, patchBorder=100,
raList=[154.10660740464786], tractOverlap=0.0)
hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig)
tract = hits_skymap[0]
sys.stderr.write("=====> DFZ 3/24/2017: mask.shape = " + str(mask.shape) + "\n")
########################################
####### Setup LSST ends
########################################
cnt = 0
def run(config, inputFiles, weightFiles=None, varianceFiles=None,
returnCalibSources=False, displayResults=[], verbose=False):
#
# Create the tasks
#
schema = afwTable.SourceTable.makeMinimalSchema()
algMetadata = dafBase.PropertyList()
isrTask = IsrTask(config=config.isr)
calibrateTask = CalibrateTask(config=config.calibrate)
sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema)
if config.doDeblend:
if SourceDeblendTask:
sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema)
else:
print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False"
config.doDeblend = False
sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement,
schema=schema, algMetadata=algMetadata)
sourceMeasurementTask.config.doApplyApCorr = 'yes'
#
# Add fields needed to identify stars while calibrating
#
keysToCopy = [(schema.addField(afwTable.Field["Flag"]("calib_detected",
"Source was detected by calibrate")), None)]
for key in calibrateTask.getCalibKeys():
keysToCopy.append((schema.addField(calibrateTask.schema.find(key).field), key))
exposureDict = {}; calibSourcesDict = {}; sourcesDict = {}
for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles):
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# read the data
#
if verbose:
print "Reading %s" % inputFile
exposure = makeExposure(inputFile, weightFile, varianceFile,
config.badPixelValue, config.variance)
#
if config.interpPlanes:
import lsst.ip.isr as ipIsr
defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes,
growFootprints=0)
isrTask.run(exposure, defects=defects)
#
# process the data
#
if config.doCalibrate:
result = calibrateTask.run(exposure)
exposure, calibSources = result.exposure, result.sources
else:
calibSources = None
if not exposure.getPsf():
calibrateTask.installInitialPsf(exposure)
exposureDict[inputFile] = exposure
calibSourcesDict[inputFile] = calibSources if returnCalibSources else None
result = sourceDetectionTask.run(tab, exposure)
sources = result.sources
sourcesDict[inputFile] = sources
if config.doDeblend:
sourceDeblendTask.run(exposure, sources, exposure.getPsf())
sourceMeasurementTask.measure(exposure, sources)
if verbose:
print "Detected %d objects" % len(sources)
propagateCalibFlags(keysToCopy, calibSources, sources)
if displayResults: # display results of processing (see also --debug argparse option)
showApertures = "showApertures".upper() in displayResults
showShapes = "showShapes".upper() in displayResults
display = afwDisplay.getDisplay(frame=1)
if algMetadata.exists("base_CircularApertureFlux_radii"):
radii = algMetadata.get("base_CircularApertureFlux_radii")
else:
radii = []
display.mtv(exposure, title=os.path.split(inputFile)[1])
with display.Buffering():
for s in sources:
xy = s.getCentroid()
display.dot('+', *xy,
ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN)
if showShapes:
display.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
if showApertures:
for radius in radii:
display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
return exposureDict, calibSourcesDict, sourcesDict
def main():
# try out one exposure
#visits = ["0288935","0288976"] #,"0289893","0289913","0289931","0289614","0289818","0289820", "0289850","0289851","0289871","0289892", "0288935","0288976","0289016","0289056","0289161","0289202","0289243","0289284","0289368","0289409","0289450","0289493","0289573","0289656"]
visits = ["0288976", "0288935"]
ccds = []
exit(0)
for i in range(1, 61):
ccds.append(i)
filterName = 'g'
DATA_PATH = "/root/extra_home/lsst_data/"
#spathprefix = "/home/dongfang/download/lsst_data/"
spathprefix = DATA_PATH + "raw/"
#calexpsloc = "/home/dongfang/download/lsst_data/calexps/"
calexpsloc = DATA_PATH + "calexps/"
#coaddloc = "/home/dongfang/download/lsst_data/coadds/"
coaddloc = DATA_PATH + "coadds/"
#mergecoaddloc = "/home/dongfang/download/lsst_data/merge/"
mergecoaddloc = DATA_PATH + "merge/"
# Characterize Image
charImageConfig = CharacterizeImageConfig()
charImage = CharacterizeImageTask()
calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False)
calibrateTask = CalibrateTask(config=calibrateConfig)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig(
projection='STG',
decList=[-4.9325280994132905],
patchInnerDimensions=[2000, 2000],
radiusList=[4.488775723429071],
pixelScale=0.333,
rotation=0.0,
patchBorder=100,
raList=[154.10660740464786],
tractOverlap=0.0)
hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig)
tract = hits_skymap[0]
coaddTempDict = {}
calibResDict = {}
f = open("log.txt", 'wb')
start = datetime.datetime.now()
#process CCDs to create calexps.
for v in visits:
for ccd in ccds:
visit = int(v)
filename = "instcal" + v + "." + str(ccd) + ".fits"
calexpfn = calexpsloc + v + "/" + filename
source = spathprefix + v + "/" + filename
exposure = afwImg.ExposureF(source)
try:
# Characterize Image
charRes = charImage.characterize(exposure,
exposureIdInfo=None,
background=None)
except:
f.write("DFZ DEBUG at charRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
# Caliberate Image
calibRes = calibrateTask.calibrate(
charRes.exposure,
exposureIdInfo=None,
background=charRes.background,
icSourceCat=None)
except:
f.write("DFZ DEBUG at calibRes: errors in visit " + v +
", ccd " + str(ccd) + "\n")
try:
#write out calexps
calibRes.exposure.writeFits(calexpfn)
#calbresDict.append((v,ccd),calibRes)
except:
f.write("DFZ DEBUG at calibRes.exposure: errors in visit " +
v + ", ccd " + str(ccd) + "\n")
end = datetime.datetime.now()
d = end - start
f.write("time for creating calexps: ")
f.write(str(d.total_seconds()))
f.write("\n")
#time for creating co-add tempexps.
start = datetime.datetime.now()
# map calexps to patch-ids
visit = visits[0]
ccdsPerPatch = []
for ccd in ccds:
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
exposure = afwImg.ExposureF(source)
bbox = exposure.getBBox()
wcs = exposure.getWcs()
corners = bbox.getCorners()
xIndexMax, yIndexMax = tract.findPatch(
wcs.pixelToSky(corners[0][0], corners[0][1])).getIndex()
xIndexMin, yIndexMin = tract.findPatch(
wcs.pixelToSky(corners[2][0], corners[2][1])).getIndex()
yy = range(yIndexMin, yIndexMax + 1)
xx = range(xIndexMin, xIndexMax + 1)
for yIdx in yy:
for xIdx in xx:
ccdsPerPatch.append((ccd, (xIdx, yIdx)))
print len(ccdsPerPatch)
#import cPickle
#cPickle.dump(open("ccdsinpatch.p",'wb'),ccdsPerPatch)
# import cPickle
# f = open("ccdsInPatch.p",'wb')
# cPickle.dump(ccdsInPatch,f)
#import cPickle
#ccdsInPatch = cPickle.load(open("ccdsInPatch.p",'rb'))
df = pd.DataFrame(ccdsPerPatch)
dfgby = df.groupby(1)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
coaddTempExpDict = {}
for visit in visits:
for a in dfgby.indices:
coaddTempExpDict[a] = {}
xInd = a[0]
yInd = a[1]
skyInfo = getSkyInfo(hits_skymap, xInd, yInd)
v = int(visit)
coaddTempExp = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddTempExp.getMaskedImage().set(
numpy.nan, afwImage.MaskU.getPlaneBitMask("NO_DATA"),
numpy.inf)
totGoodPix = 0
didSetMetadata = False
modelPsf = makeCTEConfig.modelPsf.apply(
) if makeCTEConfig.doPsfMatch else None
setInputRecorder = False
for b in dfgby.get_group(a)[0].ravel():
print a
print b
if not setInputRecorder:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
try:
inputRecorder = makeCTE.inputRecorder.makeCoaddTempExpRecorder(
v, ccdsinPatch)
except:
f.write("DFZ DEBUG at inputRecorder\n")
setInputRecorder = True
numGoodPix = 0
ccd = b
filename = "instcal" + visit + "." + str(ccd) + ".fits"
source = calexpsloc + visit + "/" + filename
calExp = afwImg.ExposureF(source)
ccdId = calExp.getId()
warpedCcdExp = makeCTE.warpAndPsfMatch.run(
calExp,
modelPsf=modelPsf,
wcs=skyInfo.wcs,
maxBBox=skyInfo.bbox).exposure
if didSetMetadata:
mimg = calExp.getMaskedImage()
mimg *= (coaddTempExp.getCalib().getFluxMag0()[0] /
calExp.getCalib().getFluxMag0()[0])
del mimg
numGoodPix = coaddUtils.copyGoodPixels(
coaddTempExp.getMaskedImage(),
warpedCcdExp.getMaskedImage(), makeCTE.getBadPixelMask())
totGoodPix += numGoodPix
if numGoodPix > 0 and not didSetMetadata:
coaddTempExp.setCalib(warpedCcdExp.getCalib())
coaddTempExp.setFilter(warpedCcdExp.getFilter())
didSetMetadata = True
inputRecorder.addCalExp(calExp, ccdId, numGoodPix)
##### End loop over ccds here:
inputRecorder.finish(coaddTempExp, totGoodPix)
if totGoodPix > 0 and didSetMetadata:
coaddTempExp.setPsf(
modelPsf if makeCTEConfig.doPsfMatch else CoaddPsf(
inputRecorder.coaddInputs.ccds, skyInfo.wcs))
coaddTempExpDict[a][v] = coaddTempExp
coaddfilename = coaddloc + visit + "/" + "instcal" + visit + "." + str(
xInd) + "_" + str(yInd) + ".fits"
coaddTempExp.writeFits(coaddfilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating co-add tempexps:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
#DFZ: stop here
exit(0)
start = datetime.datetime.now()
config = AssembleCoaddConfig()
assembleTask = AssembleCoaddTask(config=config)
mergcoadds = {}
for a in dfgby.indices:
ccdsinPatch = len(dfgby.get_group(a)[0].ravel())
xInd = a[0]
yInd = a[1]
imageScalerRes = prepareInputs(coaddTempExpDict[a].values(),
coaddTempExpDict[a].keys(),
assembleTask)
mask = None
doClip = False
if mask is None:
mask = assembleTask.getBadPixelMask()
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setNumSigmaClip(assembleTask.config.sigmaClip)
statsCtrl.setNumIter(assembleTask.config.clipIter)
statsCtrl.setAndMask(mask)
statsCtrl.setNanSafe(True)
statsCtrl.setWeighted(True)
statsCtrl.setCalcErrorFromInputVariance(True)
for plane, threshold in assembleTask.config.maskPropagationThresholds.items(
):
bit = afwImage.MaskU.getMaskPlane(plane)
statsCtrl.setMaskPropagationThreshold(bit, threshold)
if doClip:
statsFlags = afwMath.MEANCLIP
else:
statsFlags = afwMath.MEAN
coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
coaddExposure.setCalib(assembleTask.scaleZeroPoint.getCalib())
coaddExposure.getInfo().setCoaddInputs(
assembleTask.inputRecorder.makeCoaddInputs())
#remember to set metadata if you want any hope of running detection and measurement on this coadd:
#self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
#most important thing is the psf
coaddExposure.setFilter(coaddTempExpDict[a].values()[0].getFilter())
coaddInputs = coaddExposure.getInfo().getCoaddInputs()
for tempExp, weight in zip(coaddTempExpDict[a].values(),
imageScalerRes.weightList):
assembleTask.inputRecorder.addVisitToCoadd(coaddInputs, tempExp,
weight)
#takes numCcds as argument
coaddInputs.ccds.reserve(ccdsinPatch)
coaddInputs.visits.reserve(len(imageScalerRes.dataIdList))
psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs())
coaddExposure.setPsf(psf)
maskedImageList = afwImage.vectorMaskedImageF()
coaddMaskedImage = coaddExposure.getMaskedImage()
for dataId, imageScaler, exposure in zip(
imageScalerRes.dataIdList, imageScalerRes.imageScalerList,
coaddTempExpDict[a].values()):
print dataId, imageScaler, exposure
maskedImage = exposure.getMaskedImage()
imageScaler.scaleMaskedImage(maskedImage)
maskedImageList.append(maskedImage)
maskedImage = afwMath.statisticsStack(maskedImageList, statsFlags,
statsCtrl,
imageScalerRes.weightList)
coaddMaskedImage.assign(maskedImage, skyInfo.bbox)
coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(),
coaddMaskedImage.getVariance())
# write out Coadd!
mergefilename = mergecoaddloc + str(xInd) + "_" + str(yInd) + ".fits"
mergcoadds[a] = coaddExposure
coaddExposure.writeFits(mergefilename)
end = datetime.datetime.now()
d = end - start
f.write("time for creating merged co-adds:\n ")
f.write(str(d.total_seconds()))
f.write("\n")
start = datetime.datetime.now()
config = DetectCoaddSourcesConfig()
detectCoaddSources = DetectCoaddSourcesTask(config=config)
for a in dfgby.indices:
# Detect on Coadd
exp = mergcoadds[a]
detRes = detectCoaddSources.runDetection(exp, idFactory=None)
end = datetime.datetime.now()
d = end - start
f.write("time for detecting sources:\n ")
f.write(str(d.total_seconds()))
f.close()
########################################
####### Setup LSST begins
########################################
visit = 289697
ccdnum = 1
HOME_PATH = os.path.expanduser("~")
butler = dafPersist.Butler(HOME_PATH + '/lsst_data/raw')
#exposure = butler.get("instcal", visit=visit, ccdnum=ccdnum, filter='g', immediate=True)
#mask = exposure.getMaskedImage().getMask().getArray() #TODO: move this matrix to SciDB
#variance = exposure.getMaskedImage().getVariance().getArray() #TODO: move this matrix to SciDB
#filename = HOME_PATH + "/lsst_data/raw/crblasted0289697/instcal0289697.1.fits"
#fitsHeader = afwImage.readMetadata(filename)
#wcs = afwImage.makeWcs(fitsHeader)
charImage = CharacterizeImageTask()
calibrateConfig = CalibrateConfig(doPhotoCal=False, doAstrometry=False)
calibrateTask = CalibrateTask(config=calibrateConfig)
makeCTEConfig = MakeCoaddTempExpConfig()
makeCTE = MakeCoaddTempExpTask(config=makeCTEConfig)
newSkyMapConfig = skymap.discreteSkyMap.DiscreteSkyMapConfig(
projection='STG',
decList=[-4.9325280994132905],
patchInnerDimensions=[2000, 2000],
radiusList=[4.488775723429071],
pixelScale=0.333,
rotation=0.0,
patchBorder=100,
raList=[154.10660740464786],
tractOverlap=0.0)
hits_skymap = skymap.discreteSkyMap.DiscreteSkyMap(config=newSkyMapConfig)
tract = hits_skymap[0]
from EdgeRolloffWarper import EdgeRolloffWarper image_file='star_grid_warped.fits' schema = afwTable.SourceTable.makeMinimalSchema() schema.setVersion(0) # # CalibrateTask # cal_config = CalibrateTask.ConfigClass() cal_config.doAstrometry = False cal_config.doPhotoCal = False cal_config.doBackground = True calibrateTask = CalibrateTask(config=cal_config) calibrateTask.repair.config.doCosmicRay = False calibrateTask.detection.config.thresholdValue = 5 calibrateTask.detection.config.includeThresholdMultiplier = 1 calibrateTask.config.initialPsf.fwhm = 2.85 # # SourceDetectionTask # det_config = SourceDetectionTask.ConfigClass() det_config.reEstimateBackground = False detectionTask = SourceDetectionTask(config=det_config, schema=schema) #
def testComponents(self):
"""Test that we can run the first-level subtasks of ProcessCcdTasks.
This tests that we can run these subtasks from the command-line independently (they're all
CmdLineTasks) as well as directly from Python (without giving them access to a Butler).
Aside from verifying that no exceptions are raised, we simply tests that most persisted results are
present and equivalent to both in-memory results.
"""
outPath = tempfile.mkdtemp() if OutputName is None else "{}-Components".format(OutputName)
# We'll use an input butler to get data for the tasks we call from Python, but we won't ever give it
# to those tasks.
inputButler = lsst.daf.persistence.Butler(InputDir)
# Construct task instances we can use directly from Python
isrTask = IsrTask(
config=getObsTestConfig(IsrTask),
name="isr2"
)
# If we ever enable astrometry and photocal in obs_test, we'll need to pass a refObjLoader to these
# tasks. To maintain the spirit of these tests, we'd ideally have a LoadReferenceObjectsTask class
# that doesn't require a Butler. If we don't, we should construct a butler-based on outside these
# task constructors and pass the LoadReferenceObjectsTask instance to the task constructors.
charImageTask = CharacterizeImageTask(
config=getObsTestConfig(CharacterizeImageTask),
name="charImage2"
)
calibrateTask = CalibrateTask(
config=getObsTestConfig(CalibrateTask),
name="calibrate2",
icSourceSchema=charImageTask.schema
)
try:
dataId = dict(visit=1)
dataIdStrList = ["%s=%s" % (key, val) for key, val in dataId.items()]
isrResult1 = IsrTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
# We'll just use the butler to get the original image and calibration frames; it's not clear
# extending the test coverage to include that is worth it.
dataRef = inputButler.dataRef("raw", dataId=dataId)
rawExposure = dataRef.get("raw", immediate=True)
camera = dataRef.get("camera")
isrData = isrTask.readIsrData(dataRef, rawExposure)
isrResult2 = isrTask.run(
rawExposure,
bias=isrData.bias,
linearizer=isrData.linearizer,
flat=isrData.flat,
defects=isrData.defects,
fringes=isrData.fringes,
bfKernel=isrData.bfKernel,
camera=camera,
)
self.assertMaskedImagesEqual(
isrResult1.parsedCmd.butler.get("postISRCCD", dataId, immediate=True).getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
isrResult2.exposure.getMaskedImage(),
isrResult1.resultList[0].result.exposure.getMaskedImage()
)
icResult1 = CharacterizeImageTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
icResult2 = charImageTask.run(isrResult2.exposure)
self.assertMaskedImagesEqual(
icResult1.parsedCmd.butler.get("icExp", dataId, immediate=True).getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
icResult2.exposure.getMaskedImage(),
icResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
icResult1.parsedCmd.butler.get("icSrc", dataId, immediate=True),
icResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
icResult2.sourceCat,
icResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent") # since we didn't want to pass in an ExposureIdInfo, IDs disagree
)
self.assertBackgroundListsEqual(
icResult1.parsedCmd.butler.get("icExpBackground", dataId, immediate=True),
icResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
icResult2.background,
icResult1.resultList[0].result.background
)
calResult1 = CalibrateTask.parseAndRun(
args=[InputDir, "--output", outPath, "--clobber-config", "--doraise", "--id"] + dataIdStrList,
doReturnResults=True,
)
calResult2 = calibrateTask.run(
icResult2.exposure,
background=icResult2.background,
icSourceCat=icResult2.sourceCat
)
self.assertMaskedImagesEqual(
calResult1.parsedCmd.butler.get("calexp", dataId, immediate=True).getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertMaskedImagesEqual(
calResult2.exposure.getMaskedImage(),
calResult1.resultList[0].result.exposure.getMaskedImage()
)
self.assertCatalogsEqual(
calResult1.parsedCmd.butler.get("src", dataId, immediate=True),
calResult1.resultList[0].result.sourceCat
)
self.assertCatalogsEqual(
calResult2.sourceCat,
calResult1.resultList[0].result.sourceCat,
skipCols=("id", "parent")
)
self.assertBackgroundListsEqual(
calResult1.parsedCmd.butler.get("calexpBackground", dataId, immediate=True),
calResult1.resultList[0].result.background
)
self.assertBackgroundListsEqual(
calResult2.background,
calResult1.resultList[0].result.background
)
finally:
if OutputName is None:
shutil.rmtree(outPath)
else:
print("testProcessCcd.py's output data saved to %r" % (OutputName,))
#!/usr/bin/env python # # LSST Data Management System # Copyright 2016 LSST/AURA # # This product includes software developed by the # LSST Project (http://www.lsst.org/). # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # from lsst.pipe.tasks.calibrate import CalibrateTask CalibrateTask.parseAndRun()
