def testLinearCombinationKernel(self): """Test LinearCombinationKernel using a set of delta basis functions """ kWidth = 3 kHeight = 2 pol = pexPolicy.Policy() additionalData = dafBase.PropertySet() loc = dafPersist.LogicalLocation("tests/data/kernel5.boost") persistence = dafPersist.Persistence.getPersistence(pol) # create list of kernels basisImArrList = [] kVec = afwMath.KernelList() for row in range(kHeight): for col in range(kWidth): kernel = afwMath.DeltaFunctionKernel(kWidth, kHeight, afwGeom.Point2I(col, row)) basisImage = afwImage.ImageD(kernel.getDimensions()) kernel.computeImage(basisImage, True) basisImArrList.append(basisImage.getArray().transpose().copy()) kVec.append(kernel) kParams = [0.0]*len(kVec) k = afwMath.LinearCombinationKernel(kVec, kParams) for ii in range(len(kVec)): kParams = [0.0]*len(kVec) kParams[ii] = 1.0 k.setKernelParameters(kParams) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage("XmlStorage", loc) storageList.append(storage) persistence.persist(k, storageList, additionalData) storageList2 = dafPersist.StorageList() storage2 = persistence.getRetrieveStorage("XmlStorage", loc) storageList2.append(storage2) x = persistence.unsafeRetrieve("LinearCombinationKernel", storageList2, additionalData) k2 = afwMath.LinearCombinationKernel.swigConvert(x) self.kernelCheck(k, k2) kIm = afwImage.ImageD(k2.getDimensions()) k2.computeImage(kIm, True) kImArr = kIm.getArray().transpose() if not numpy.allclose(kImArr, basisImArrList[ii]): self.fail("%s = %s != %s for the %s'th basis kernel" % \ (k2.__class__.__name__, kImArr, basisImArrList[ii], ii))
def testFixedKernel(self): """Test FixedKernel using a ramp function """ kWidth = 5 kHeight = 6 inArr = numpy.arange(kWidth * kHeight, dtype=float) inArr.shape = [kWidth, kHeight] inImage = afwImage.ImageD(afwGeom.Extent2I(kWidth, kHeight)) for row in range(inImage.getHeight()): for col in range(inImage.getWidth()): inImage.set(col, row, inArr[col, row]) k = afwMath.FixedKernel(inImage) pol = pexPolicy.Policy() additionalData = dafBase.PropertySet() loc = dafPersist.LogicalLocation("tests/data/kernel1.boost") persistence = dafPersist.Persistence.getPersistence(pol) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage("XmlStorage", loc) storageList.append(storage) persistence.persist(k, storageList, additionalData) storageList2 = dafPersist.StorageList() storage2 = persistence.getRetrieveStorage("XmlStorage", loc) storageList2.append(storage2) x = persistence.unsafeRetrieve("FixedKernel", storageList2, additionalData) k2 = afwMath.FixedKernel.swigConvert(x) self.kernelCheck(k, k2) outImage = afwImage.ImageD(k2.getDimensions()) k2.computeImage(outImage, False) outArr = outImage.getArray().transpose() if not numpy.allclose(inArr, outArr): self.fail("%s = %s != %s (not normalized)" % \ (k2.__class__.__name__, inArr, outArr)) normInArr = inArr / inArr.sum() normOutImage = afwImage.ImageD(k2.getDimensions()) k2.computeImage(normOutImage, True) normOutArr = normOutImage.getArray().transpose() if not numpy.allclose(normOutArr, normInArr): self.fail("%s = %s != %s (normalized)" % \ (k2.__class__.__name__, normInArr, normOutArr))
def testAnalyticKernel(self): """Test AnalyticKernel using a Gaussian function """ kWidth = 5 kHeight = 8 pol = pexPolicy.Policy() additionalData = dafBase.PropertySet() loc = dafPersist.LogicalLocation("tests/data/kernel2.boost") persistence = dafPersist.Persistence.getPersistence(pol) gaussFunc = afwMath.GaussianFunction2D(1.0, 1.0, 0.0) k = afwMath.AnalyticKernel(kWidth, kHeight, gaussFunc) fArr = numpy.zeros(shape=[k.getWidth(), k.getHeight()], dtype=float) for xsigma in (0.1, 1.0, 3.0): for ysigma in (0.1, 1.0, 3.0): for angle in (0.0, 0.4, 1.1): gaussFunc.setParameters((xsigma, ysigma, angle)) # compute array of function values and normalize for row in range(k.getHeight()): y = row - k.getCtrY() for col in range(k.getWidth()): x = col - k.getCtrX() fArr[col, row] = gaussFunc(x, y) fArr /= fArr.sum() k.setKernelParameters((xsigma, ysigma, angle)) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage("XmlStorage", loc) storageList.append(storage) persistence.persist(k, storageList, additionalData) storageList2 = dafPersist.StorageList() storage2 = persistence.getRetrieveStorage("XmlStorage", loc) storageList2.append(storage2) x = persistence.unsafeRetrieve("AnalyticKernel", storageList2, additionalData) k2 = afwMath.AnalyticKernel.swigConvert(x) self.kernelCheck(k, k2) kImage = afwImage.ImageD(k2.getDimensions()) k2.computeImage(kImage, True) kArr = kImage.getArray().transpose() if not numpy.allclose(fArr, kArr): self.fail("%s = %s != %s for xsigma=%s, ysigma=%s" % \ (k2.__class__.__name__, kArr, fArr, xsigma, ysigma))
def testBoostPersistence(self): """Persist the image using boost""" with utilsTests.getTempFilePath(".boost") as boostFilePath: logicalLocation = dafPers.LogicalLocation(boostFilePath) storage = self.persistence.getPersistStorage("BoostStorage", logicalLocation) storageList = dafPers.StorageList([storage]) self.persistence.persist(self.image, storageList, self.additionalData) # Retrieve it again storage = self.persistence.getRetrieveStorage("BoostStorage", logicalLocation) storageList = dafPers.StorageList([storage]) pers2Ptr = self.persistence.unsafeRetrieve("ImageF", storageList, self.additionalData) image2 = afwImage.ImageF.swigConvert(pers2Ptr) # Check the resulting Image self.checkImages(self.image, image2)
def testFitsPersistence(self): """Test persisting to FITS""" # Set up the LogicalLocation. logicalLocation = dafPers.LogicalLocation(self.infile) # Create a FitsStorage and put it in a StorageList. storage = self.persistence.getRetrieveStorage("FitsStorage", logicalLocation) storageList = dafPers.StorageList([storage]) # Let's do the retrieval! maskedImage2 = afwImage.MaskedImageF.swigConvert( \ self.persistence.unsafeRetrieve("MaskedImageF", storageList, self.additionalData)) # Check the resulting MaskedImage self.checkImages(self.maskedImage, maskedImage2)
def testFitsPersistence(self): """Test unpersisting from FITS""" # Set up the LogicalLocation. logicalLocation = dafPers.LogicalLocation( os.path.join("tests", "data", "HSC-0908120-056-small.fits")) # Create a FitsStorage and put it in a StorageList. storage = self.persistence.getRetrieveStorage("FitsStorage", logicalLocation) storageList = dafPers.StorageList([storage]) # Let's do the retrieval! propertyList = self.persistence.unsafeRetrieve("PropertyList", storageList, None) self.assertEqual(propertyList.get("AR_HDU"), 5)
def testDeltaFunctionKernel(self): """Test DeltaFunctionKernel """ pol = pexPolicy.Policy() additionalData = dafBase.PropertySet() loc = dafPersist.LogicalLocation( os.path.join(testPath, "data", "kernel3.boost")) persistence = dafPersist.Persistence.getPersistence(pol) for kWidth in range(1, 4): for kHeight in range(1, 4): for activeCol in range(kWidth): for activeRow in range(kHeight): kernel = afwMath.DeltaFunctionKernel( kWidth, kHeight, afwGeom.Point2I(activeCol, activeRow)) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage( "XmlStorage", loc) storageList.append(storage) persistence.persist(kernel, storageList, additionalData) storageList2 = dafPersist.StorageList() storage2 = persistence.getRetrieveStorage( "XmlStorage", loc) storageList2.append(storage2) x = persistence.unsafeRetrieve("DeltaFunctionKernel", storageList2, additionalData) k2 = afwMath.DeltaFunctionKernel.swigConvert(x) self.kernelCheck(kernel, k2) self.assertEqual(kernel.getPixel(), k2.getPixel()) kImage = afwImage.ImageD(k2.getDimensions()) kSum = k2.computeImage(kImage, False) self.assertEqual(kSum, 1.0) kArr = kImage.getArray().transpose() self.assertEqual(kArr[activeCol, activeRow], 1.0) kArr[activeCol, activeRow] = 0.0 self.assertEqual(kArr.sum(), 0.0)
def testBoostPersistenceU16(self): """Persist a U16 image using boost""" with utilsTests.getTempFilePath(".boost") as boostFilePath: logicalLocation = dafPers.LogicalLocation(boostFilePath) storage = self.persistence.getPersistStorage("BoostStorage", logicalLocation) storageList = dafPers.StorageList([storage]) # # Read a U16 image # self.image = self.image.Factory(os.path.join(dataDir, "data", "small_MI.fits")) self.persistence.persist(self.image, storageList, self.additionalData) # Retrieve it again storage = self.persistence.getRetrieveStorage("BoostStorage", logicalLocation) storageList = dafPers.StorageList([storage]) pers2Ptr = self.persistence.unsafeRetrieve("ImageF", storageList, self.additionalData) image2 = afwImage.ImageF.swigConvert(pers2Ptr) # Check the resulting Image self.checkImages(self.image, image2)
def unpersistPsf(xmlPath): """Read a PSF from an XML file""" # Set up persistence object pol = pexPolicy.Policy() persistence = dafPersist.Persistence.getPersistence(pol) # Where is the file on disk? Make a storage object loc = dafPersist.LogicalLocation(xmlPath) storageList = dafPersist.StorageList() storage = persistence.getRetrieveStorage('XmlStorage', loc) storageList.append(storage) # Capture any associated metadata metadata = dafBase.PropertySet() # Unpersist the object; you need to say which object in storage to grab persistable = persistence.retrieve('pcaPsf', storageList, metadata) # Cast to a PSF model psf = measAlg.PSF_swigConvert(persistable) return psf
def testBoostPersistence(self): """Persist the image using boost""" boostFilePath = "image.boost" logicalLocation = dafPers.LogicalLocation(boostFilePath) storage = self.persistence.getPersistStorage("BoostStorage", logicalLocation) storageList = dafPers.StorageList([storage]) self.persistence.persist(self.image, storageList, self.additionalData) # Retrieve it again storage = self.persistence.getRetrieveStorage("BoostStorage", logicalLocation) storageList = dafPers.StorageList([storage]) pers2Ptr = self.persistence.unsafeRetrieve("ImageF", storageList, self.additionalData) image2 = afwImage.ImageF.swigConvert(pers2Ptr) # Check the resulting Image self.checkImages(self.image, image2) # Delete the boost-persisted image os.remove(boostFilePath)
def test3(self): """ Test PropertySet persistence to database with policy mapping itemName to database table name and mapping property keys to table columns. """ dp = dafBase.PropertySet() dp.addInt("i", 3) dp.addDouble("d", 3.4) dp.addString("v", "LastOne") dp.addBool("b", True) dp.addLongLong("I", 9998887776) dp.addFloat("f", 1.414) pol = pexPolicy.Policy() itemPol = pexPolicy.Policy() itemPol.set("TableName", "Persistence_Test_2") itemPol.add("KeyList", "floatField=f") itemPol.add("KeyList", "int64Field=I") itemPol.add("KeyList", "boolField=b") itemPol.add("KeyList", "varcharField=v") itemPol.add("KeyList", "doubleField=d") itemPol.add("KeyList", "intField=i") pol.set("Formatter.PropertySet.testItem", itemPol) additionalData = dafBase.PropertySet() additionalData.add("itemName", "testItem") loc = dafPersist.LogicalLocation("mysql://{}:{}/test".format(HOST, PORT)) persistence = dafPersist.Persistence.getPersistence(pol) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage("DbStorage", loc) storageList.append(storage) persistence.persist(dp, storageList, additionalData)
def testSVLinearCombinationKernel(self): """Test a spatially varying LinearCombinationKernel """ kWidth = 3 kHeight = 2 pol = pexPolicy.Policy() additionalData = dafBase.PropertySet() loc = dafPersist.LogicalLocation( os.path.join(testPath, "data", "kernel6.boost")) persistence = dafPersist.Persistence.getPersistence(pol) # create image arrays for the basis kernels basisImArrList = [] imArr = np.zeros((kWidth, kHeight), dtype=float) imArr += 0.1 imArr[kWidth // 2, :] = 0.9 basisImArrList.append(imArr) imArr = np.zeros((kWidth, kHeight), dtype=float) imArr += 0.2 imArr[:, kHeight // 2] = 0.8 basisImArrList.append(imArr) # create a list of basis kernels from the images kVec = [] for basisImArr in basisImArrList: basisImage = afwImage.makeImageFromArray( basisImArr.transpose().copy()) kernel = afwMath.FixedKernel(basisImage) kVec.append(kernel) # create spatially varying linear combination kernel spFunc = afwMath.PolynomialFunction2D(1) # spatial parameters are a list of entries, one per kernel parameter; # each entry is a list of spatial parameters sParams = ( (0.0, 1.0, 0.0), (0.0, 0.0, 1.0), ) k = afwMath.LinearCombinationKernel(kVec, spFunc) k.setSpatialParameters(sParams) storageList = dafPersist.StorageList() storage = persistence.getPersistStorage("XmlStorage", loc) storageList.append(storage) persistence.persist(k, storageList, additionalData) storageList2 = dafPersist.StorageList() storage2 = persistence.getRetrieveStorage("XmlStorage", loc) storageList2.append(storage2) k2 = persistence.unsafeRetrieve("LinearCombinationKernel", storageList2, additionalData) self.kernelCheck(k, k2) kImage = afwImage.ImageD(afwGeom.Extent2I(kWidth, kHeight)) for colPos, rowPos, coeff0, coeff1 in [ (0.0, 0.0, 0.0, 0.0), (1.0, 0.0, 1.0, 0.0), (0.0, 1.0, 0.0, 1.0), (1.0, 1.0, 1.0, 1.0), (0.5, 0.5, 0.5, 0.5), ]: k2.computeImage(kImage, False, colPos, rowPos) kImArr = kImage.getArray().transpose() refKImArr = (basisImArrList[0] * coeff0) + \ (basisImArrList[1] * coeff1) if not np.allclose(kImArr, refKImArr): self.fail( "%s = %s != %s at colPos=%s, rowPos=%s" % (k2.__class__.__name__, kImArr, refKImArr, colPos, rowPos))
def _massagePolicy(self): loc = persistence.LogicalLocation(self._policy.get('filterTableLocation')) self._policy.set('filterTableLocation', loc.locString())
def _setupRegistry(self, name, description, path, policy, policyKey, storage, searchParents=True, posixIfNoSql=True): """Set up a registry (usually SQLite3), trying a number of possible paths. Parameters ---------- name : string Name of registry. description: `str` Description of registry (for log messages) path : string Path for registry. policy : string Policy that contains the registry name, used if path is None. policyKey : string Key in policy for registry path. storage : Storage subclass Repository Storage to look in. searchParents : bool, optional True if the search for a registry should follow any Butler v1 _parent symlinks. posixIfNoSql : bool, optional If an sqlite registry is not found, will create a posix registry if this is True. Returns ------- lsst.daf.persistence.Registry Registry object """ if path is None and policyKey in policy: path = dafPersist.LogicalLocation(policy[policyKey]).locString() if os.path.isabs(path): raise RuntimeError("Policy should not indicate an absolute path for registry.") if not storage.exists(path): newPath = storage.instanceSearch(path) newPath = newPath[0] if newPath is not None and len(newPath) else None if newPath is None: self.log.warn("Unable to locate registry at policy path (also looked in root): %s", path) path = newPath else: self.log.warn("Unable to locate registry at policy path: %s", path) path = None # Old Butler API was to indicate the registry WITH the repo folder, New Butler expects the registry to # be in the repo folder. To support Old API, check to see if path starts with root, and if so, strip # root from path. Currently only works with PosixStorage try: root = storage.root if path and (path.startswith(root)): path = path[len(root + '/'):] except AttributeError: pass # determine if there is an sqlite registry and if not, try the posix registry. registry = None def search(filename, description): """Search for file in storage Parameters ---------- filename : `str` Filename to search for description : `str` Description of file, for error message. Returns ------- path : `str` or `None` Path to file, or None """ result = storage.instanceSearch(filename) if result: return result[0] self.log.debug("Unable to locate %s: %s", description, filename) return None # Search for a suitable registry database if path is None: path = search("%s.pgsql" % name, "%s in root" % description) if path is None: path = search("%s.sqlite3" % name, "%s in root" % description) if path is None: path = search(os.path.join(".", "%s.sqlite3" % name), "%s in current dir" % description) if path is not None: if not storage.exists(path): newPath = storage.instanceSearch(path) newPath = newPath[0] if newPath is not None and len(newPath) else None if newPath is not None: path = newPath localFileObj = storage.getLocalFile(path) self.log.info("Loading %s registry from %s", description, localFileObj.name) registry = dafPersist.Registry.create(localFileObj.name) localFileObj.close() elif not registry and posixIfNoSql: try: self.log.info("Loading Posix %s registry from %s", description, storage.root) registry = dafPersist.PosixRegistry(storage.root) except Exception: registry = None return registry
def __init__(self, root=None, registry=None, calibRoot=None, calibRegistry=None, provided=None, parentRegistry=None, repositoryCfg=None): """Initialize the CameraMapper. Parameters ---------- policy : daf_persistence.Policy, Can also be pexPolicy.Policy, only for backward compatibility. Policy with per-camera defaults already merged. repositoryDir : string Policy repository for the subclassing module (obtained with getRepositoryPath() on the per-camera default dictionary). root : string, optional Path to the root directory for data. registry : string, optional Path to registry with data's metadata. calibRoot : string, optional Root directory for calibrations. calibRegistry : string, optional Path to registry with calibrations' metadata. provided : list of string, optional Keys provided by the mapper. parentRegistry : Registry subclass, optional Registry from a parent repository that may be used to look up data's metadata. repositoryCfg : daf_persistence.RepositoryCfg or None, optional The configuration information for the repository this mapper is being used with. """ policyFile = Policy.defaultPolicyFile("obs_hsc_sims", "HscSimsMapper.yaml", "policy") policy = Policy(policyFile) dafPersist.Mapper.__init__(self) self.log = lsstLog.Log.getLogger("HscSimsMapper") if root: self.root = root elif repositoryCfg: self.root = repositoryCfg.root else: self.root = None if isinstance(policy, pexPolicy.Policy): policy = dafPersist.Policy(policy) repoPolicy = repositoryCfg.policy if repositoryCfg else None if repoPolicy is not None: policy.update(repoPolicy) # Don't load the default policy from obs_base # defaultPolicyFile = dafPersist.Policy.defaultPolicyFile("obs_base", # "MapperDictionary.paf", # "policy") # dictPolicy = dafPersist.Policy(defaultPolicyFile) # policy.merge(dictPolicy) # Levels self.levels = dict() if 'levels' in policy: levelsPolicy = policy['levels'] for key in levelsPolicy.names(True): self.levels[key] = set(levelsPolicy.asArray(key)) self.defaultLevel = policy['defaultLevel'] self.defaultSubLevels = dict() if 'defaultSubLevels' in policy: self.defaultSubLevels = policy['defaultSubLevels'] # Root directories if root is None: root = "." root = dafPersist.LogicalLocation(root).locString() self.rootStorage = dafPersist.Storage.makeFromURI(uri=root) # If the calibRoot is passed in, use that. If not and it's indicated in # the policy, use that. And otherwise, the calibs are in the regular # root. # If the location indicated by the calib root does not exist, do not # create it. calibStorage = None if calibRoot is not None: calibRoot = dafPersist.Storage.absolutePath(root, calibRoot) calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot, create=False) else: calibRoot = policy.get('calibRoot', None) if calibRoot: calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot, create=False) if calibStorage is None: calibStorage = self.rootStorage self.root = root # Registries self.registry = self._setupRegistry("registry", "exposure", registry, policy, "registryPath", self.rootStorage, searchParents=False, posixIfNoSql=True) if not self.registry: self.registry = parentRegistry needCalibRegistry = policy.get('needCalibRegistry', None) if needCalibRegistry: if calibStorage: self.calibRegistry = self._setupRegistry("calibRegistry", "calib", calibRegistry, policy, "calibRegistryPath", calibStorage, posixIfNoSql=False) # NB never use posix for calibs else: raise RuntimeError( "'needCalibRegistry' is true in Policy, but was unable to locate a repo at " + "calibRoot ivar:%s or policy['calibRoot']:%s" % (calibRoot, policy.get('calibRoot', None))) else: self.calibRegistry = None # Dict of valid keys and their value types self.keyDict = dict() self._initMappings(policy, self.rootStorage, calibStorage, provided=None) self._initWriteRecipes() # Camera geometry # #self.cameraDataLocation = None # path to camera geometry config file # #self.camera = self._makeCamera(policy=policy, repositoryDir=repositoryDir) # Defect registry and root. Defects are stored with the camera and the registry is loaded from the # camera package, which is on the local filesystem. # #self.defectRegistry = None # #if 'defects' in policy: # # self.defectPath = os.path.join(repositoryDir, policy['defects']) # # defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3") # # self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation) # Filter translation table self.filters = None
def fetchOrbitIdsAndEphems(dbLogicalLocation, sliceId, numSlices, mjd, deltaMJD=1.): """ Fetch the orbit Id of all known moving objects from day-MOPS together with their precomputed ephemerides at int(mjd)-deltaMJD, int(mjd) and int(mjd)+deltaMJD. @param dbLogicalLocation: pointer to the DB. @param sliceId: slice ID. @param numSlices: total number of slices. @param mjd: MJD of the exposure (UTC). @param deltaMJD: temporal distance betweeb successive ephemerides. Return [(internal_orbitId: Ephemeris obj), ] sorted by mjd """ # Init the persistance middleware. if (RIDICOLOUSLY_VERBOSE): t3 = time.time() db = dafPer.DbStorage() # Connect to the DB. loc = dafPer.LogicalLocation(dbLogicalLocation) db.setRetrieveLocation(loc) if (RIDICOLOUSLY_VERBOSE): logit(' %.02fs: connect to DB' % (time.time() - t3)) # Prepare the query. if (RIDICOLOUSLY_VERBOSE): t3 = time.time() deltaMJD = abs(deltaMJD) mjdMin = mjd - deltaMJD mjdMax = mjd + deltaMJD # TODO: handle different MovingObject versions. Meaning choose the highest # version. Not needed for DC3a. where = 'mjd >= %f and mjd <= %f' % (mjdMin, mjdMax) # Poor man parallelism ;-) # if(numSlices > 0): if (numSlices > 1): where += ' and movingObjectId %% %d = %d' % (numSlices, sliceId) db.startTransaction() db.setTableForQuery('_tmpl_mops_Ephemeris') db.setQueryWhere(where) db.outColumn('movingObjectId') db.outColumn('movingObjectVersion') db.outColumn('mjd') db.outColumn('ra') db.outColumn('decl') db.outColumn('mag') db.outColumn('smaa') db.outColumn('smia') db.outColumn('pa') db.orderBy('movingObjectId') db.orderBy('mjd') if (RIDICOLOUSLY_VERBOSE): logit(' %.02fs: prepare SQL' % (time.time() - t3)) # Execute the query. if (RIDICOLOUSLY_VERBOSE): t3 = time.time() db.query() if (RIDICOLOUSLY_VERBOSE): logit(' %.02fs: exec SQL' % (time.time() - t3)) # Fetch the results. if (RIDICOLOUSLY_VERBOSE): t0 = time.time() t1 = 0 t2 = 0 res = [] while db.next(): if (RIDICOLOUSLY_VERBOSE): tt2 = time.time() movingObjectId = db.getColumnByPosInt64(0) movingObjectVersion = db.getColumnByPosInt64(1) ephem = ( movingObjectId, # movingObjectId movingObjectVersion, # movingObjectVersion db.getColumnByPosDouble(2), # mjd db.getColumnByPosDouble(3), # ra db.getColumnByPosDouble(4), # decl db.getColumnByPosDouble(5), # mag db.getColumnByPosDouble(6), # smaa db.getColumnByPosDouble(7), # smia db.getColumnByPosDouble(8)) # pa if (RIDICOLOUSLY_VERBOSE): t2 += time.time() - tt2 # We now create a new temp id made by concatenating the movingobject id # and its version. It will only be used internally. # res= [(new_orbit_id, Ephemeris obj), ...] if (RIDICOLOUSLY_VERBOSE): tt1 = time.time() res.append(('%d-%d' % (movingObjectId, movingObjectVersion), ephem)) if (RIDICOLOUSLY_VERBOSE): t1 += time.time() - tt1 # We are done with the query. db.finishQuery() if (RIDICOLOUSLY_VERBOSE): logit(' %.02fs: fetch res' % (time.time() - t0 - t1 - t2)) logit(' %.02fs: Ephemeris()' % (t2)) logit(' %.02fs: results.append()' % (t1)) return (res)
#!/usr/bin/env python import sys import lsst.daf.persistence as persistence if(len(sys.argv) != 2): sys.stderr.write('usage: cleanall.py <database name>\n') sys.stderr.flush() sys.exit(1) database = sys.argv[1] db = persistence.DbStorage() db.setRetrieveLocation(persistence.LogicalLocation('mysql://localhost:3306/%s' %(database))) db.startTransaction() db.executeSql('delete from DIASourceIDTonight') db.executeSql('delete from mops_Tracklet') db.executeSql('delete from mops_TrackletsToDIASource') db.executeSql('delete from mops_TracksToTracklet') db.endTransaction() sys.exit(0)
def _output(stage, policy, clipboard, log): """Perform the actual persistence. @param stage The stage requesting output. @param policy The policy for the stage. @param clipboard The clipboard for the stage. The persisted objects are taken from this. @param log A logger for messages. """ if not policy.exists('parameters.outputItems'): # Propagate the clipboard to the output queue, but otherwise # do nothing. log.log(Log.WARN, "No outputItems found") return mainAdditionalData = lsst.pex.harness.Utils.createAdditionalData( stage, policy, clipboard) # Create a persistence object using policy, if present. if policy.exists('parameters.persistence'): persistencePolicy = pexPolicy.Policy( policy.getPolicy('parameters.persistence')) else: persistencePolicy = pexPolicy.Policy() persistence = dafPersist.Persistence.getPersistence(persistencePolicy) # Iterate over items in OutputItems policy. outputPolicy = policy.getPolicy('parameters.outputItems') itemNames = outputPolicy.policyNames(True) somethingWasOutput = False for item in itemNames: additionalData = mainAdditionalData.deepCopy() itemPolicy = outputPolicy.getPolicy(item) # Skip the item if it is not required and is not present. itemRequired = itemPolicy.exists('required') and \ itemPolicy.getBool('required') if not clipboard.contains(item): if itemRequired: raise RuntimeError, 'Missing output item: ' + item else: continue itemData = clipboard.get(item) # Add the item name to the additionalData. additionalData.set('itemName', item) if itemPolicy.exists('datasetId'): dsPolicy = itemPolicy.getPolicy('datasetId') ds = Dataset(dsPolicy.get('datasetType')) ds.ids = {} if dsPolicy.exists('set'): setPolicy = dsPolicy.getPolicy('set') for param in setPolicy.paramNames(): ds.ids[param] = setPolicy.get(param) additionalData.set(param, setPolicy.get(param)) if dsPolicy.exists('fromJobIdentity'): jobIdentity = clipboard.get( policy.get('inputKeys.jobIdentity')) for id in dsPolicy.getStringArray('fromJobIdentity'): ds.ids[id] = jobIdentity[id] additionalData.set(id, jobIdentity[id]) outputKey = policy.get('outputKeys.outputDatasets') dsList = clipboard.get(outputKey) if dsList is None: dsList = [] clipboard.put(outputKey, dsList) dsList.append(ds) if stage.butler is not None: # Use the butler to figure out storage and locations. # Write Using Butler iolog = BlockTimingLog(log, "write_using_butler", Log.INFO - 1) iolog.start("persisting %s as %s with keys %s" % (item, ds.type, ds.ids)) stage.butler.put(itemData, ds.type, dataId=ds.ids) iolog.done() somethingWasOutput = True continue # Get the item's StoragePolicy. if itemPolicy.isArray('storagePolicy'): policyList = itemPolicy.getPolicyArray('storagePolicy') else: policyList = [] policyList.append(itemPolicy.getPolicy('storagePolicy')) # Create a list of Storages for the item based on policy. storageList = dafPersist.StorageList() for storagePolicy in policyList: storageName = storagePolicy.getString('storage') location = storagePolicy.getString('location') logLoc = dafPersist.LogicalLocation(location, additionalData) log.log(Log.INFO - 1, "persisting %s to %s" % (item, logLoc.locString())) additionalData.add('StorageLocation.' + storageName, logLoc.locString()) mainAdditionalData.add('StorageLocation.' + storageName, logLoc.locString()) storage = persistence.getPersistStorage(storageName, logLoc) storageList.append(storage) # Persist the item. # Write Without Butler iolog = BlockTimingLog(log, "write_without_butler", Log.INFO - 1) iolog.start("persisting %s" % (item, )) if hasattr(itemData, '__deref__'): persistence.persist(itemData.__deref__(), storageList, additionalData) else: persistence.persist(itemData, storageList, additionalData) iolog.done() somethingWasOutput = True if not somethingWasOutput: log.log(Log.WARN, "No items were output")