def testBoostPersistenceU16(self):
"""Persist a U16 image using boost"""
with lsst.utils.tests.getTempFilePath(".boost") as boostFilePath:
logicalLocation = dafPers.LogicalLocation(boostFilePath)
storage = self.persistence.getPersistStorage(
"BoostStorage", logicalLocation)
storageList = dafPers.StorageList([storage])
#
# Read a U16 image
#
self.image = self.image.Factory(
os.path.join(dataDir, "data", "small_MI.fits"))
self.persistence.persist(self.image, storageList,
self.additionalData)
# Retrieve it again
storage = self.persistence.getRetrieveStorage(
"BoostStorage", logicalLocation)
storageList = dafPers.StorageList([storage])
pers2Ptr = self.persistence.unsafeRetrieve("ImageF", storageList,
self.additionalData)
image2 = afwImage.ImageF.swigConvert(pers2Ptr)
# Check the resulting Image
self.checkImages(self.image, image2)
def testSetCtr(self):
"""Test setCtrCol/Row"""
kWidth = 3
kHeight = 4
pol = pexPolicy.Policy()
additionalData = dafBase.PropertySet()
loc = dafPersist.LogicalLocation(
os.path.join(testPath, "data", "kernel7.boost"))
persistence = dafPersist.Persistence.getPersistence(pol)
gaussFunc = afwMath.GaussianFunction2D(1.0, 1.0, 0.0)
k = afwMath.AnalyticKernel(kWidth, kHeight, gaussFunc)
for xCtr in range(kWidth):
k.setCtrX(xCtr)
for yCtr in range(kHeight):
k.setCtrY(yCtr)
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("XmlStorage", loc)
storageList.append(storage)
persistence.persist(k, storageList, additionalData)
storageList2 = dafPersist.StorageList()
storage2 = persistence.getRetrieveStorage("XmlStorage", loc)
storageList2.append(storage2)
k2 = persistence.unsafeRetrieve("AnalyticKernel", storageList2,
additionalData)
self.kernelCheck(k, k2)
self.assertEqual(k2.getCtrX(), xCtr)
self.assertEqual(k2.getCtrY(), yCtr)
def testPersistence(self):
if persistence.DbAuth.available("lsst10.ncsa.uiuc.edu", "3306"):
pol = policy.Policy()
root = "Formatter.PersistableIdPairVector"
pol.set(root + ".IdPair.templateTableName", "_tmpl_IdPair")
pol.set(root + ".IdPair.tableNamePattern", "_tmp_v%(visitId)_IP")
pers = persistence.Persistence.getPersistence(pol)
loc = persistence.LogicalLocation(
"mysql://lsst10.ncsa.uiuc.edu:3306/test")
ps = base.PropertySet()
ps.setInt("visitId",
int(time.clock()) * 16384 + random.randint(0, 16383))
ps.setString("itemName", "IdPair")
stl = persistence.StorageList()
stl.push_back(pers.getPersistStorage("DbStorage", loc))
pers.persist(ap.PersistableIdPairVector(self.vec), stl, ps)
stl = persistence.StorageList()
stl.push_back(pers.getRetrieveStorage("DbStorage", loc))
res = ap.PersistableIdPairVector.swigConvert(
pers.unsafeRetrieve("PersistableIdPairVector", stl, ps))
db = persistence.DbStorage()
db.setPersistLocation(loc)
db.startTransaction()
db.dropTable(detection.getTableName(pol.getPolicy(root), ps))
db.endTransaction()
assert _seqEqual(res.getIdPairs(), self.vec)
def persistCovMaps(covMaps, outputDir):
"""Persists coverage maps for up to 6 LSST filters (ugrizy) and
produces an all-filter coverage map by summing the individual maps.
Output files are automatically gzipped (they tend to compress well).
"""
if not os.path.exists(outputDir):
os.makedirs(outputDir)
persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy())
for covMap, filter in izip(covMaps, "ugrizy"):
# Write out coverage map
outPath = os.path.join(outputDir, '%sCovMap.fits' % filter)
storageList = dafPersist.StorageList()
storageList.append(
persistence.getPersistStorage("FitsStorage",
dafPersist.LogicalLocation(outPath)))
persistence.persist(covMap, storageList, dafBase.PropertySet())
subprocess.check_call(['gzip', '-f', outPath])
allCovMap = covMaps.pop()
allCovMapImg = allCovMap.getImage()
for covMap in covMaps:
allCovMapImg += covMap.getImage()
outPath = os.path.join(outputDir, 'allCovMap.fits')
storageList = dafPersist.StorageList()
storageList.append(
persistence.getPersistStorage("FitsStorage",
dafPersist.LogicalLocation(outPath)))
persistence.persist(allCovMap, storageList, dafBase.PropertySet())
subprocess.check_call(['gzip', '-f', outPath])
def roundTripPsf(key, psf):
global psfFileNum
pol = policy.Policy()
additionalData = dafBase.PropertySet()
if psfFileNum % 2 == 1:
storageType = "Boost"
else:
storageType = "Xml"
loc = dafPersist.LogicalLocation("tests/data/psf%d-%d.%s" %
(psfFileNum, key, storageType))
psfFileNum += 1
persistence = dafPersist.Persistence.getPersistence(pol)
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("%sStorage" % (storageType), loc)
storageList.append(storage)
persistence.persist(psf, storageList, additionalData)
storageList2 = dafPersist.StorageList()
storage2 = persistence.getRetrieveStorage("%sStorage" % (storageType), loc)
storageList2.append(storage2)
psf2 = persistence.unsafeRetrieve("Psf", storageList2, additionalData)
return psf2
def testBoostPersistenceU16(self):
"""Persist a U16 image using boost"""
boostFilePath = "image.boost"
logicalLocation = dafPers.LogicalLocation(boostFilePath)
storage = self.persistence.getPersistStorage(
"BoostStorage", logicalLocation)
storageList = dafPers.StorageList([storage])
#
# Read a U16 image
#
self.image = self.im.Factory(
os.path.join(dataDir, "small_MI_img.fits"))
self.persistence.persist(self.image, storageList,
self.additionalData)
# Retrieve it again
storage = self.persistence.getRetrieveStorage(
"BoostStorage", logicalLocation)
storageList = dafPers.StorageList([storage])
pers2Ptr = self.persistence.unsafeRetrieve("ImageF", storageList,
self.additionalData)
image2 = afwImage.ImageF.swigConvert(pers2Ptr)
# Check the resulting Image
self.checkImages(self.image, image2)
# Delete the boost-persisted image
os.remove(boostFilePath)
def testPersistence1(self):
dp = dafBase.PropertySet()
dp.addInt("foo", 3)
pol = lsst.pex.policy.Policy()
additionalData = dafBase.PropertySet()
additionalData.addInt("sliceId", 5)
loc = dafPersist.LogicalLocation(os.path.join(ROOT, "data/test.boost"))
persistence = dafPersist.Persistence.getPersistence(pol)
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("BoostStorage", loc)
storageList.append(storage)
persistence.persist(dp, storageList, additionalData)
storageList = dafPersist.StorageList()
storage = persistence.getRetrieveStorage("BoostStorage", loc)
storageList.append(storage)
# pybind11 automatically returns most derived type
rdp = persistence.unsafeRetrieve("PropertySet", storageList,
additionalData)
self.assertEqual(rdp.nameCount(), 1)
self.assertTrue(rdp.exists("foo"))
self.assertEqual(rdp.getInt("foo"), 3)
def testPersistence(self):
if dafPers.DbAuth.available("lsst10.ncsa.uiuc.edu", "3306"):
pol = pexPolicy.Policy()
pol.set(
"Formatter.PersistableMovingObjectPredictionVector.TestPreds.templateTableName",
"_tmpl_mops_Prediction")
pol.set(
"Formatter.PersistableMovingObjectPredictionVector.TestPreds.tableNamePattern",
"_tmp_v%(visitId)_Preds")
pers = dafPers.Persistence.getPersistence(pol)
loc = dafPers.LogicalLocation(
"mysql://lsst10.ncsa.uiuc.edu:3306/test")
props = dafBase.PropertySet()
props.addInt("visitId",
int(time.clock()) * 16384 + random.randint(0, 16383))
props.addInt("sliceId", 0)
props.addInt("numSlices", 1)
props.addString("itemName", "TestPreds")
stl = dafPers.StorageList()
stl.push_back(pers.getPersistStorage("DbStorage", loc))
pers.persist(mops.PersistableMopsPredVec(self.mpv1), stl, props)
stl = dafPers.StorageList()
stl.push_back(pers.getRetrieveStorage("DbStorage", loc))
persistable = pers.unsafeRetrieve(
"PersistableMovingObjectPredictionVector", stl, props)
res = mops.PersistableMopsPredVec.swigConvert(persistable)
afwDet.dropAllSliceTables(
loc,
pol.getPolicy(
"Formatter.PersistableMovingObjectPredictionVector"),
props)
checkMopsPredEqual(res.getPredictions(), self.mpv1)
else:
print "skipping database tests"
def testSeparableKernel(self):
"""Test SeparableKernel using a Gaussian function
"""
kWidth = 5
kHeight = 8
pol = pexPolicy.Policy()
additionalData = dafBase.PropertySet()
loc = dafPersist.LogicalLocation(
os.path.join(testPath, "data", "kernel4.boost"))
persistence = dafPersist.Persistence.getPersistence(pol)
gaussFunc1 = afwMath.GaussianFunction1D(1.0)
k = afwMath.SeparableKernel(kWidth, kHeight, gaussFunc1, gaussFunc1)
fArr = np.zeros(shape=[k.getWidth(), k.getHeight()], dtype=float)
np.zeros(shape=[k.getWidth(), k.getHeight()], dtype=float)
gaussFunc = afwMath.GaussianFunction2D(1.0, 1.0, 0.0)
for xsigma in (0.1, 1.0, 3.0):
gaussFunc1.setParameters((xsigma, ))
for ysigma in (0.1, 1.0, 3.0):
gaussFunc.setParameters((xsigma, ysigma, 0.0))
# compute array of function values and normalize
for row in range(k.getHeight()):
y = row - k.getCtrY()
for col in range(k.getWidth()):
x = col - k.getCtrX()
fArr[col, row] = gaussFunc(x, y)
fArr /= fArr.sum()
k.setKernelParameters((xsigma, ysigma))
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("XmlStorage", loc)
storageList.append(storage)
persistence.persist(k, storageList, additionalData)
storageList2 = dafPersist.StorageList()
storage2 = persistence.getRetrieveStorage("XmlStorage", loc)
storageList2.append(storage2)
x = persistence.unsafeRetrieve("SeparableKernel", storageList2,
additionalData)
k2 = afwMath.SeparableKernel.swigConvert(x)
self.kernelCheck(k, k2)
kImage = afwImage.ImageD(k2.getDimensions())
k2.computeImage(kImage, True)
kArr = kImage.getArray().transpose()
if not np.allclose(fArr, kArr):
self.fail(
"%s = %s != %s for xsigma=%s, ysigma=%s" %
(k2.__class__.__name__, kArr, fArr, xsigma, ysigma))
def testLinearCombinationKernel(self):
"""Test LinearCombinationKernel using a set of delta basis functions
"""
kWidth = 3
kHeight = 2
pol = pexPolicy.Policy()
additionalData = dafBase.PropertySet()
loc = dafPersist.LogicalLocation(
os.path.join(testPath, "data", "kernel5.boost"))
persistence = dafPersist.Persistence.getPersistence(pol)
# create list of kernels
basisImArrList = []
kVec = afwMath.KernelList()
for row in range(kHeight):
for col in range(kWidth):
kernel = afwMath.DeltaFunctionKernel(kWidth, kHeight,
afwGeom.Point2I(col, row))
basisImage = afwImage.ImageD(kernel.getDimensions())
kernel.computeImage(basisImage, True)
basisImArrList.append(basisImage.getArray().transpose().copy())
kVec.append(kernel)
kParams = [0.0] * len(kVec)
k = afwMath.LinearCombinationKernel(kVec, kParams)
for ii in range(len(kVec)):
kParams = [0.0] * len(kVec)
kParams[ii] = 1.0
k.setKernelParameters(kParams)
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("XmlStorage", loc)
storageList.append(storage)
persistence.persist(k, storageList, additionalData)
storageList2 = dafPersist.StorageList()
storage2 = persistence.getRetrieveStorage("XmlStorage", loc)
storageList2.append(storage2)
x = persistence.unsafeRetrieve("LinearCombinationKernel",
storageList2, additionalData)
k2 = afwMath.LinearCombinationKernel.swigConvert(x)
self.kernelCheck(k, k2)
kIm = afwImage.ImageD(k2.getDimensions())
k2.computeImage(kIm, True)
kImArr = kIm.getArray().transpose()
if not np.allclose(kImArr, basisImArrList[ii]):
self.fail(
"%s = %s != %s for the %s'th basis kernel" %
(k2.__class__.__name__, kImArr, basisImArrList[ii], ii))
def testFixedKernel(self):
"""Test FixedKernel using a ramp function
"""
kWidth = 5
kHeight = 6
inArr = np.arange(kWidth * kHeight, dtype=float)
inArr.shape = [kWidth, kHeight]
inImage = afwImage.ImageD(afwGeom.Extent2I(kWidth, kHeight))
for row in range(inImage.getHeight()):
for col in range(inImage.getWidth()):
inImage.set(col, row, inArr[col, row])
k = afwMath.FixedKernel(inImage)
pol = pexPolicy.Policy()
additionalData = dafBase.PropertySet()
loc = dafPersist.LogicalLocation(
os.path.join(testPath, "data", "kernel1.boost"))
persistence = dafPersist.Persistence.getPersistence(pol)
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("XmlStorage", loc)
storageList.append(storage)
persistence.persist(k, storageList, additionalData)
storageList2 = dafPersist.StorageList()
storage2 = persistence.getRetrieveStorage("XmlStorage", loc)
storageList2.append(storage2)
x = persistence.unsafeRetrieve("FixedKernel", storageList2,
additionalData)
k2 = afwMath.FixedKernel.swigConvert(x)
self.kernelCheck(k, k2)
outImage = afwImage.ImageD(k2.getDimensions())
k2.computeImage(outImage, False)
outArr = outImage.getArray().transpose()
if not np.allclose(inArr, outArr):
self.fail("%s = %s != %s (not normalized)" %
(k2.__class__.__name__, inArr, outArr))
normInArr = inArr / inArr.sum()
normOutImage = afwImage.ImageD(k2.getDimensions())
k2.computeImage(normOutImage, True)
normOutArr = normOutImage.getArray().transpose()
if not np.allclose(normOutArr, normInArr):
self.fail("%s = %s != %s (normalized)" %
(k2.__class__.__name__, normInArr, normOutArr))
def testBoostPersistence(self):
"""Persist the image using boost"""
with utilsTests.getTempFilePath(".boost") as boostFilePath:
logicalLocation = dafPers.LogicalLocation(boostFilePath)
storage = self.persistence.getPersistStorage("BoostStorage", logicalLocation)
storageList = dafPers.StorageList([storage])
self.persistence.persist(self.image, storageList, self.additionalData)
# Retrieve it again
storage = self.persistence.getRetrieveStorage("BoostStorage", logicalLocation)
storageList = dafPers.StorageList([storage])
pers2Ptr = self.persistence.unsafeRetrieve("ImageF", storageList, self.additionalData)
image2 = afwImage.ImageF.swigConvert(pers2Ptr)
# Check the resulting Image
self.checkImages(self.image, image2)
def _read(item, cppType, pythonType, storageInfo, additionalData, persistence,
log):
# Create a list of Storages for the item based on policy.
storageList = dafPersist.StorageList()
for storageName, location in storageInfo:
logLoc = dafPersist.LogicalLocation(location, additionalData)
if not logLoc.locString().startswith("mysql:"):
_waitForPath(logLoc.locString(), log)
log.log(Log.INFO - 1,
"loading %s from %s" % (item, logLoc.locString()))
storage = persistence.getRetrieveStorage(storageName, logLoc)
storageList.append(storage)
iolog = BlockTimingLog(log, "read_without_butler", Log.INFO - 1)
iolog.start("load of %s" % (item, ))
# Retrieve the item.
itemData = persistence.unsafeRetrieve(cppType, storageList, additionalData)
# Cast the SWIGged Persistable to a more useful type.
cvt = getattr(pythonType, "swigConvert")
finalItem = cvt(itemData)
iolog.done()
# If Persistable and subclasses are NOT wrapped using SWIG_SHARED_PTR,
# then one must make sure that the wrapper for the useful type owns
# the pointer (rather than the wrapper for the original Persistable).
# The following lines accomplish this:
#itemData.this.disown()
#finalItem.this.acquire()
# Put the item on the clipboard
return finalItem
def test2(self):
"""
Test PropertySet persistence to database with policy mapping itemName to
database table name.
"""
dp = dafBase.PropertySet()
dp.addInt("intField", 2)
dp.addDouble("doubleField", 2.3)
dp.addString("varcharField", "gnitseT")
dp.addBool("boolField", False)
dp.addLongLong("int64Field", 9988776655)
dp.addFloat("floatField", 2.718)
pol = pexPolicy.Policy()
itemPol = pexPolicy.Policy()
itemPol.set("TableName", "Persistence_Test_2")
pol.set("Formatter.PropertySet.testItem", itemPol)
additionalData = dafBase.PropertySet()
additionalData.add("itemName", "testItem")
loc = dafPersist.LogicalLocation("mysql://{}:{}/test".format(HOST, PORT))
persistence = dafPersist.Persistence.getPersistence(pol)
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("DbStorage", loc)
storageList.append(storage)
persistence.persist(dp, storageList, additionalData)
def test1(self):
"""
Test PropertySet persistence to database without policy.
"""
dp = dafBase.PropertySet()
dp.addInt("intField", 1)
dp.addDouble("doubleField", 1.2)
dp.addString("varcharField", "Testing")
dp.addBool("boolField", True)
dp.addLongLong("int64Field", 9876543210)
dp.addFloat("floatField", 3.14)
pol = pexPolicy.Policy()
additionalData = dafBase.PropertySet()
additionalData.add("itemName", "Persistence_Test_2")
loc = dafPersist.LogicalLocation("mysql://{}:{}/test".format(HOST, PORT))
persistence = dafPersist.Persistence.getPersistence(pol)
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("DbStorage", loc)
storageList.append(storage)
persistence.persist(dp, storageList, additionalData)
def readSourceSet(boostFile):
loc = dafPersist.LogicalLocation(boostFile)
storageList = dafPersist.StorageList()
additionalData = dafBase.PropertySet()
persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy())
storageList.append(persistence.getRetrieveStorage("BoostStorage", loc))
psv = persistence.unsafeRetrieve("PersistableSourceVector", storageList, additionalData)
return psv.getSources()
def sourceset_read_boost(fn):
loc = dafPersist.LogicalLocation(fn)
storageList = dafPersist.StorageList()
additionalData = dafBase.PropertySet()
persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy())
storageList.append(persistence.getRetrieveStorage("BoostStorage", loc))
psvptr = persistence.unsafeRetrieve("PersistableSourceVector", storageList, additionalData)
print(psvptr)
psv = afwDet.PersistableSourceVector.swigConvert(psvptr)
return psv.getSources()
def pcapsf_read_boost(fn):
print('# Reading', fn, end=' ')
loc = dafPersist.LogicalLocation(fn)
storageList = dafPersist.StorageList()
additionalData = dafBase.PropertySet()
persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy())
storageList.append(persistence.getRetrieveStorage("BoostStorage", loc))
psf = persistence.unsafeRetrieve("Psf", storageList, additionalData)
return psf
def testDeltaFunctionKernel(self):
"""Test DeltaFunctionKernel
"""
pol = pexPolicy.Policy()
additionalData = dafBase.PropertySet()
loc = dafPersist.LogicalLocation(
os.path.join(testPath, "data", "kernel3.boost"))
persistence = dafPersist.Persistence.getPersistence(pol)
for kWidth in range(1, 4):
for kHeight in range(1, 4):
for activeCol in range(kWidth):
for activeRow in range(kHeight):
kernel = afwMath.DeltaFunctionKernel(
kWidth, kHeight,
afwGeom.Point2I(activeCol, activeRow))
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage(
"XmlStorage", loc)
storageList.append(storage)
persistence.persist(kernel, storageList,
additionalData)
storageList2 = dafPersist.StorageList()
storage2 = persistence.getRetrieveStorage(
"XmlStorage", loc)
storageList2.append(storage2)
x = persistence.unsafeRetrieve("DeltaFunctionKernel",
storageList2,
additionalData)
k2 = afwMath.DeltaFunctionKernel.swigConvert(x)
self.kernelCheck(kernel, k2)
self.assertEqual(kernel.getPixel(), k2.getPixel())
kImage = afwImage.ImageD(k2.getDimensions())
kSum = k2.computeImage(kImage, False)
self.assertEqual(kSum, 1.0)
kArr = kImage.getArray().transpose()
self.assertEqual(kArr[activeCol, activeRow], 1.0)
kArr[activeCol, activeRow] = 0.0
self.assertEqual(kArr.sum(), 0.0)
def testBoostPersistence(self):
"""Persist the MaskedImage using boost"""
miPath = os.path.join("tests", "data", "Dest")
logicalLocation = dafPers.LogicalLocation(miPath)
storage = self.persistence.getPersistStorage("BoostStorage",
logicalLocation)
storageList = dafPers.StorageList([storage])
self.persistence.persist(self.maskedImage, storageList,
self.additionalData)
def pcapsf_read_boost(fn):
import lsst.meas.algorithms as measAlgorithms # needed to register pcaPsf formatter
print('# Reading', fn, end=' ')
loc = dafPersist.LogicalLocation(fn)
storageList = dafPersist.StorageList()
additionalData = dafBase.PropertySet()
persistence = dafPersist.Persistence.getPersistence(pexPolicy.Policy())
storageList.append(persistence.getRetrieveStorage("BoostStorage", loc))
psf = persistence.unsafeRetrieve("Psf", storageList, additionalData)
return psf
def testBoostPersistence(self):
"""Persist the MaskedImage using boost"""
miPath = os.path.join("tests", "data", "Dest")
logicalLocation = dafPers.LogicalLocation(miPath)
storage = self.persistence.getPersistStorage("BoostStorage", logicalLocation)
storageList = dafPers.StorageList([storage])
try:
self.persistence.persist(self.maskedImage, storageList, self.additionalData)
except pexExceptions.LsstCppException, e:
print e.args[0].what()
raise
def testBoostPersistence(self):
"""Persist the image using boost"""
boostFilePath = "image.boost"
logicalLocation = dafPers.LogicalLocation(boostFilePath)
storage = self.persistence.getPersistStorage("BoostStorage",
logicalLocation)
storageList = dafPers.StorageList([storage])
self.persistence.persist(self.image, storageList, self.additionalData)
# Retrieve it again
storage = self.persistence.getRetrieveStorage("BoostStorage",
logicalLocation)
storageList = dafPers.StorageList([storage])
pers2Ptr = self.persistence.unsafeRetrieve("ImageF", storageList,
self.additionalData)
image2 = afwImage.ImageF.swigConvert(pers2Ptr)
# Check the resulting Image
self.checkImages(self.image, image2)
# Delete the boost-persisted image
os.remove(boostFilePath)
def testFitsPersistence(self):
"""Test persisting to FITS"""
# Set up the LogicalLocation.
logicalLocation = dafPers.LogicalLocation(self.infile)
# Create a FitsStorage and put it in a StorageList.
storage = self.persistence.getRetrieveStorage("FitsStorage", logicalLocation)
storageList = dafPers.StorageList([storage])
# Let's do the retrieval!
maskedImage2 = afwImage.MaskedImageF.swigConvert( \
self.persistence.unsafeRetrieve("MaskedImageF", storageList, self.additionalData))
# Check the resulting MaskedImage
self.checkImages(self.maskedImage, maskedImage2)
def testFitsPersistence(self):
"""Test unpersisting from FITS"""
# Set up the LogicalLocation.
logicalLocation = dafPers.LogicalLocation(
os.path.join("tests", "data", "HSC-0908120-056-small.fits"))
# Create a FitsStorage and put it in a StorageList.
storage = self.persistence.getRetrieveStorage("FitsStorage",
logicalLocation)
storageList = dafPers.StorageList([storage])
# Let's do the retrieval!
propertyList = self.persistence.unsafeRetrieve("PropertyList",
storageList, None)
self.assertEqual(propertyList.get("AR_HDU"), 5)
def unpersistPsf(xmlPath):
"""Read a PSF from an XML file"""
# Set up persistence object
pol = pexPolicy.Policy()
persistence = dafPersist.Persistence.getPersistence(pol)
# Where is the file on disk? Make a storage object
loc = dafPersist.LogicalLocation(xmlPath)
storageList = dafPersist.StorageList()
storage = persistence.getRetrieveStorage('XmlStorage', loc)
storageList.append(storage)
# Capture any associated metadata
metadata = dafBase.PropertySet()
# Unpersist the object; you need to say which object in storage to grab
persistable = persistence.retrieve('pcaPsf', storageList, metadata)
# Cast to a PSF model
psf = measAlg.PSF_swigConvert(persistable)
return psf
def test3(self):
"""
Test PropertySet persistence to database with policy mapping itemName to
database table name and mapping property keys to table columns.
"""
dp = dafBase.PropertySet()
dp.addInt("i", 3)
dp.addDouble("d", 3.4)
dp.addString("v", "LastOne")
dp.addBool("b", True)
dp.addLongLong("I", 9998887776)
dp.addFloat("f", 1.414)
pol = pexPolicy.Policy()
itemPol = pexPolicy.Policy()
itemPol.set("TableName", "Persistence_Test_2")
itemPol.add("KeyList", "floatField=f")
itemPol.add("KeyList", "int64Field=I")
itemPol.add("KeyList", "boolField=b")
itemPol.add("KeyList", "varcharField=v")
itemPol.add("KeyList", "doubleField=d")
itemPol.add("KeyList", "intField=i")
pol.set("Formatter.PropertySet.testItem", itemPol)
additionalData = dafBase.PropertySet()
additionalData.add("itemName", "testItem")
loc = dafPersist.LogicalLocation("mysql://{}:{}/test".format(HOST, PORT))
persistence = dafPersist.Persistence.getPersistence(pol)
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("DbStorage", loc)
storageList.append(storage)
persistence.persist(dp, storageList, additionalData)
def testSVLinearCombinationKernel(self):
"""Test a spatially varying LinearCombinationKernel
"""
kWidth = 3
kHeight = 2
pol = pexPolicy.Policy()
additionalData = dafBase.PropertySet()
loc = dafPersist.LogicalLocation(
os.path.join(testPath, "data", "kernel6.boost"))
persistence = dafPersist.Persistence.getPersistence(pol)
# create image arrays for the basis kernels
basisImArrList = []
imArr = np.zeros((kWidth, kHeight), dtype=float)
imArr += 0.1
imArr[kWidth // 2, :] = 0.9
basisImArrList.append(imArr)
imArr = np.zeros((kWidth, kHeight), dtype=float)
imArr += 0.2
imArr[:, kHeight // 2] = 0.8
basisImArrList.append(imArr)
# create a list of basis kernels from the images
kVec = []
for basisImArr in basisImArrList:
basisImage = afwImage.makeImageFromArray(
basisImArr.transpose().copy())
kernel = afwMath.FixedKernel(basisImage)
kVec.append(kernel)
# create spatially varying linear combination kernel
spFunc = afwMath.PolynomialFunction2D(1)
# spatial parameters are a list of entries, one per kernel parameter;
# each entry is a list of spatial parameters
sParams = (
(0.0, 1.0, 0.0),
(0.0, 0.0, 1.0),
)
k = afwMath.LinearCombinationKernel(kVec, spFunc)
k.setSpatialParameters(sParams)
storageList = dafPersist.StorageList()
storage = persistence.getPersistStorage("XmlStorage", loc)
storageList.append(storage)
persistence.persist(k, storageList, additionalData)
storageList2 = dafPersist.StorageList()
storage2 = persistence.getRetrieveStorage("XmlStorage", loc)
storageList2.append(storage2)
k2 = persistence.unsafeRetrieve("LinearCombinationKernel",
storageList2, additionalData)
self.kernelCheck(k, k2)
kImage = afwImage.ImageD(afwGeom.Extent2I(kWidth, kHeight))
for colPos, rowPos, coeff0, coeff1 in [
(0.0, 0.0, 0.0, 0.0),
(1.0, 0.0, 1.0, 0.0),
(0.0, 1.0, 0.0, 1.0),
(1.0, 1.0, 1.0, 1.0),
(0.5, 0.5, 0.5, 0.5),
]:
k2.computeImage(kImage, False, colPos, rowPos)
kImArr = kImage.getArray().transpose()
refKImArr = (basisImArrList[0] * coeff0) + \
(basisImArrList[1] * coeff1)
if not np.allclose(kImArr, refKImArr):
self.fail(
"%s = %s != %s at colPos=%s, rowPos=%s" %
(k2.__class__.__name__, kImArr, refKImArr, colPos, rowPos))
def _output(stage, policy, clipboard, log):
"""Perform the actual persistence.
@param stage The stage requesting output.
@param policy The policy for the stage.
@param clipboard The clipboard for the stage. The persisted objects are taken from this.
@param log A logger for messages.
"""
if not policy.exists('parameters.outputItems'):
# Propagate the clipboard to the output queue, but otherwise
# do nothing.
log.log(Log.WARN, "No outputItems found")
return
mainAdditionalData = lsst.pex.harness.Utils.createAdditionalData(
stage, policy, clipboard)
# Create a persistence object using policy, if present.
if policy.exists('parameters.persistence'):
persistencePolicy = pexPolicy.Policy(
policy.getPolicy('parameters.persistence'))
else:
persistencePolicy = pexPolicy.Policy()
persistence = dafPersist.Persistence.getPersistence(persistencePolicy)
# Iterate over items in OutputItems policy.
outputPolicy = policy.getPolicy('parameters.outputItems')
itemNames = outputPolicy.policyNames(True)
somethingWasOutput = False
for item in itemNames:
additionalData = mainAdditionalData.deepCopy()
itemPolicy = outputPolicy.getPolicy(item)
# Skip the item if it is not required and is not present.
itemRequired = itemPolicy.exists('required') and \
itemPolicy.getBool('required')
if not clipboard.contains(item):
if itemRequired:
raise RuntimeError, 'Missing output item: ' + item
else:
continue
itemData = clipboard.get(item)
# Add the item name to the additionalData.
additionalData.set('itemName', item)
if itemPolicy.exists('datasetId'):
dsPolicy = itemPolicy.getPolicy('datasetId')
ds = Dataset(dsPolicy.get('datasetType'))
ds.ids = {}
if dsPolicy.exists('set'):
setPolicy = dsPolicy.getPolicy('set')
for param in setPolicy.paramNames():
ds.ids[param] = setPolicy.get(param)
additionalData.set(param, setPolicy.get(param))
if dsPolicy.exists('fromJobIdentity'):
jobIdentity = clipboard.get(
policy.get('inputKeys.jobIdentity'))
for id in dsPolicy.getStringArray('fromJobIdentity'):
ds.ids[id] = jobIdentity[id]
additionalData.set(id, jobIdentity[id])
outputKey = policy.get('outputKeys.outputDatasets')
dsList = clipboard.get(outputKey)
if dsList is None:
dsList = []
clipboard.put(outputKey, dsList)
dsList.append(ds)
if stage.butler is not None:
# Use the butler to figure out storage and locations.
# Write Using Butler
iolog = BlockTimingLog(log, "write_using_butler", Log.INFO - 1)
iolog.start("persisting %s as %s with keys %s" %
(item, ds.type, ds.ids))
stage.butler.put(itemData, ds.type, dataId=ds.ids)
iolog.done()
somethingWasOutput = True
continue
# Get the item's StoragePolicy.
if itemPolicy.isArray('storagePolicy'):
policyList = itemPolicy.getPolicyArray('storagePolicy')
else:
policyList = []
policyList.append(itemPolicy.getPolicy('storagePolicy'))
# Create a list of Storages for the item based on policy.
storageList = dafPersist.StorageList()
for storagePolicy in policyList:
storageName = storagePolicy.getString('storage')
location = storagePolicy.getString('location')
logLoc = dafPersist.LogicalLocation(location, additionalData)
log.log(Log.INFO - 1,
"persisting %s to %s" % (item, logLoc.locString()))
additionalData.add('StorageLocation.' + storageName,
logLoc.locString())
mainAdditionalData.add('StorageLocation.' + storageName,
logLoc.locString())
storage = persistence.getPersistStorage(storageName, logLoc)
storageList.append(storage)
# Persist the item.
# Write Without Butler
iolog = BlockTimingLog(log, "write_without_butler", Log.INFO - 1)
iolog.start("persisting %s" % (item, ))
if hasattr(itemData, '__deref__'):
persistence.persist(itemData.__deref__(), storageList,
additionalData)
else:
persistence.persist(itemData, storageList, additionalData)
iolog.done()
somethingWasOutput = True
if not somethingWasOutput:
log.log(Log.WARN, "No items were output")
