def instanceSearch(self, path):
"""Search for the given path in this storage instance.
If the path contains an HDU indicator (a number in brackets before the
dot, e.g. 'foo.fits[1]', this will be stripped when searching and so
will match filenames without the HDU indicator, e.g. 'foo.fits'. The
path returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
Parameters
----------
path : string
A filename (and optionally prefix path) to search for within root.
Returns
-------
string or None
The location that was found, or None if no location was found.
"""
strippedPath = path
if strippedPath.endswith(']'):
strippedPath = strippedPath[:strippedPath.rfind('[')]
location = dafPersist.ButlerLocation(pythonType=None,
cppType=None,
storageName=None,
locationList=[strippedPath],
dataId={},
mapper=None,
storage=None)
return bool(self.exists(location))
def map_x(self, dataId, write):
path = "foo%(ccd)d.pickle" % dataId
if not write:
path = "parent/" + path
return dafPersist.ButlerLocation(
"lsst.afw.image.BBox", "lsst::afw::image::BBox", "PickleStorage",
path, {}, self, dafPersist.Storage.makeFromURI(os.getcwd()))
def map_str(self, dataId, write):
path = os.path.join(self.root, 'raw')
path = os.path.join(path, 'raw_v' + str(dataId['str']) + '_f' + dataId['filter'] + '.fits.gz')
if os.path.exists(path):
return dp.ButlerLocation(str, None, 'PickleStorage', path, dataId,
self, self.storage)
return None
def putRepositoryCfg(cls, cfg, loc=None):
"""Serialize a RepositoryCfg to a location.
When loc == cfg.root, the RepositoryCfg is to be written at the root
location of the repository. In that case, root is not written, it is
implicit in the location of the cfg. This allows the cfg to move from
machine to machine without modification.
Parameters
----------
cfg : RepositoryCfg instance
The RepositoryCfg to be serailized.
loc : string, optional
The URI location (can be relative path) to write the RepositoryCfg.
If loc is None, the location will be read from the root parameter
of loc.
Returns
-------
None
"""
storage = dafPersist.Storage.makeFromURI(
cfg.root if loc is None else loc, create=True)
location = dafPersist.ButlerLocation(
pythonType=dafPersist.RepositoryCfg,
cppType=None,
storageName=None,
locationList=None,
dataId={},
mapper=None,
storage=storage,
usedDataId=None,
datasetType=None)
storage.write(location, cfg)
def map_raw(self, dataId, write):
python = 'astropy.io.fits.HDUList'
persistable = None
storage = 'FitsStorage'
path = 'butlerAlias/data/input/raw/raw_v' + \
str(dataId['visit']) + '_f' + dataId['filter'] + '.fits.gz'
return dafPersist.ButlerLocation(python, persistable, storage, path,
dataId, self, dafPersist.Storage.makeFromURI(ROOT))
def map_obj(self, dataId, write):
loc = dafPersist.ButlerLocation(pythonType=MyTestObject,
cppType=None,
storageName=None,
locationList=['testname'],
dataId={},
mapper=self,
storage=self.storage)
return loc
def map_table(self, dataId, write):
loc = dafPersist.ButlerLocation(pythonType=lsst.afw.table.BaseCatalog,
cppType=None,
storageName=None,
locationList=['testname'],
dataId={},
mapper=self,
storage=self.rootStorage)
return loc
def map_pickled(self, dataId, write):
python = 'dict'
persistable = None
storage = 'PickleStorage'
path = os.path.join(self.root, 'raw')
path = os.path.join(path, 'pickled_v' + str(dataId['visit']) + '_f' + dataId['filter'] + '.fits.gz')
if write or os.path.exists(path):
return dp.ButlerLocation(python, persistable, storage, path,
dataId, self, self.storage)
def map_raw(self, dataId, write):
python = 'astropy.io.fits.HDUList'
persistable = None
storage = 'PickleStorage'
path = os.path.join(self.root, 'raw')
path = os.path.join(path, 'raw_v' + str(dataId['visit']) + '_f' + dataId['filter'] + '.fits.gz')
if os.path.exists(path):
return dp.ButlerLocation(python, persistable, storage, path,
dataId, self, self.storage)
return None
def map_str(self, dataId, write):
template = "strfile_%(strId)s.pickle"
path = template % dataId
if not write:
if not self.storage.exists(path):
return None
location = self.storage.locationWithRoot(path)
return dp.ButlerLocation(pythonType=PosixPickleStringHanlder,
cppType=None,
storageName='PickleStorage',
locationList=location,
dataId=dataId,
mapper=self,
storage=self.storage)
def map_defects(self, dataId, write=False):
"""Map defects dataset.
Returns
-------
`lsst.daf.butler.ButlerLocation`
Minimal ButlerLocation containing just the locationList field
(just enough information that bypass_defects can use it).
"""
defectFitsPath = self._defectLookup(dataId=dataId)
if defectFitsPath is None:
raise RuntimeError("No defects available for dataId=%s" % (dataId,))
return dafPersist.ButlerLocation(None, None, None, defectFitsPath,
dataId, self,
storage=self.rootStorage)
def instanceSearch(self, path):
"""Search for the given path in this storage instance.
In this StorageInterface `path` is the name of a table within the connected database.
Parameters
----------
path : string
The name of a table within the connected database.
Returns
-------
bool
True if the table exists, else false.
"""
location = dafPersist.ButlerLocation(pythonType=None, cppType=None, storageName=None,
locationList=[path], dataId={}, mapper=None, storage=None)
return bool(self.exists(location))
def test_copy(self):
repoLocation = self._getS3URI('test_copy')
storage = S3Storage(uri=repoLocation, create=True)
loc = dafPersist.ButlerLocation(pythonType=MyTestObject,
cppType=None,
storageName=None,
locationList=['testname'],
dataId={},
mapper=self,
storage=storage)
testObj = MyTestObject('foo')
storage.write(loc, testObj)
storage.copyFile('testname', 'testname_copy')
storage.read(loc)
reloadedObj = storage.read(loc)
self.assertEqual(testObj, reloadedObj[0])
loc.locationList = ['testname_copy']
copiedObj = storage.read(loc)
self.assertEqual(testObj, copiedObj[0])
def getRepositoryCfg(cls, uri):
"""Get a persisted RepositoryCfg
Parameters
----------
uri : URI or path to a RepositoryCfg
Description
Returns
-------
A RepositoryCfg instance or None
"""
storage = dafPersist.Storage.makeFromURI(uri)
location = dafPersist.ButlerLocation(
pythonType=dafPersist.RepositoryCfg,
cppType=None,
storageName=None,
locationList=None,
dataId={},
mapper=None,
storage=storage,
usedDataId=None,
datasetType=None)
return storage.read(location)
def map(self, datasetType, dataId, write=False):
path = self.templates[datasetType] % dataId
return dafPersist.ButlerLocation(
None, None, "PickleStorage", path, {}, self,
dafPersist.Storage.makeFromURI(self.root))
def _initMappings(self, policy, rootStorage=None, calibStorage=None, provided=None, use_default=True):
"""Initialize mappings
For each of the dataset types that we want to be able to read, there are
methods that can be created to support them:
* map_<dataset> : determine the path for dataset
* std_<dataset> : standardize the retrieved dataset
* bypass_<dataset> : retrieve the dataset (bypassing the usual retrieval machinery)
* query_<dataset> : query the registry
Besides the dataset types explicitly listed in the policy, we create
additional, derived datasets for additional conveniences, e.g., reading
the header of an image, retrieving only the size of a catalog.
Parameters
----------
policy : `lsst.daf.persistence.Policy`
Policy with per-camera defaults already merged
rootStorage : `Storage subclass instance`
Interface to persisted repository data.
calibRoot : `Storage subclass instance`
Interface to persisted calib repository data
provided : `list` of `str`
Keys provided by the mapper
use_default : `bool`
Load default camera mappings
"""
# Sub-dictionaries (for exposure/calibration/dataset types)
imgMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
"obs_base", "ImageMappingDictionary.paf", "policy"))
expMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
"obs_base", "ExposureMappingDictionary.paf", "policy"))
calMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
"obs_base", "CalibrationMappingDictionary.paf", "policy"))
dsMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
"obs_base", "DatasetMappingDictionary.paf", "policy"))
# Mappings
mappingList = (
("images", imgMappingPolicy, ImageMapping),
("exposures", expMappingPolicy, ExposureMapping),
("calibrations", calMappingPolicy, CalibrationMapping),
("datasets", dsMappingPolicy, DatasetMapping)
)
self.mappings = dict()
for name, defPolicy, cls in mappingList:
if name in policy:
datasets = policy[name]
# Centrally-defined datasets
defaultsPath = os.path.join(getPackageDir("obs_base"), "policy", name + ".yaml")
if os.path.exists(defaultsPath) and use_default:
datasets.merge(dafPersist.Policy(defaultsPath))
mappings = dict()
setattr(self, name, mappings)
for datasetType in datasets.names(True):
subPolicy = datasets[datasetType]
subPolicy.merge(defPolicy)
if not hasattr(self, "map_" + datasetType) and 'composite' in subPolicy:
def compositeClosure(dataId, write=False, mapper=None, mapping=None,
subPolicy=subPolicy):
components = subPolicy.get('composite')
assembler = subPolicy['assembler'] if 'assembler' in subPolicy else None
disassembler = subPolicy['disassembler'] if 'disassembler' in subPolicy else None
python = subPolicy['python']
butlerComposite = dafPersist.ButlerComposite(assembler=assembler,
disassembler=disassembler,
python=python,
dataId=dataId,
mapper=self)
for name, component in components.items():
butlerComposite.add(id=name,
datasetType=component.get('datasetType'),
setter=component.get('setter', None),
getter=component.get('getter', None),
subset=component.get('subset', False),
inputOnly=component.get('inputOnly', False))
return butlerComposite
setattr(self, "map_" + datasetType, compositeClosure)
# for now at least, don't set up any other handling for this dataset type.
continue
if name == "calibrations":
mapping = cls(datasetType, subPolicy, self.registry, self.calibRegistry, calibStorage,
provided=provided, dataRoot=rootStorage)
else:
mapping = cls(datasetType, subPolicy, self.registry, rootStorage, provided=provided)
self.keyDict.update(mapping.keys())
mappings[datasetType] = mapping
self.mappings[datasetType] = mapping
if not hasattr(self, "map_" + datasetType):
def mapClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
return mapping.map(mapper, dataId, write)
setattr(self, "map_" + datasetType, mapClosure)
if not hasattr(self, "query_" + datasetType):
def queryClosure(format, dataId, mapping=mapping):
return mapping.lookup(format, dataId)
setattr(self, "query_" + datasetType, queryClosure)
if hasattr(mapping, "standardize") and not hasattr(self, "std_" + datasetType):
def stdClosure(item, dataId, mapper=weakref.proxy(self), mapping=mapping):
return mapping.standardize(mapper, item, dataId)
setattr(self, "std_" + datasetType, stdClosure)
def setMethods(suffix, mapImpl=None, bypassImpl=None, queryImpl=None):
"""Set convenience methods on CameraMapper"""
mapName = "map_" + datasetType + "_" + suffix
bypassName = "bypass_" + datasetType + "_" + suffix
queryName = "query_" + datasetType + "_" + suffix
if not hasattr(self, mapName):
setattr(self, mapName, mapImpl or getattr(self, "map_" + datasetType))
if not hasattr(self, bypassName):
if bypassImpl is None and hasattr(self, "bypass_" + datasetType):
bypassImpl = getattr(self, "bypass_" + datasetType)
if bypassImpl is not None:
setattr(self, bypassName, bypassImpl)
if not hasattr(self, queryName):
setattr(self, queryName, queryImpl or getattr(self, "query_" + datasetType))
# Filename of dataset
setMethods("filename", bypassImpl=lambda datasetType, pythonType, location, dataId:
[os.path.join(location.getStorage().root, p) for p in location.getLocations()])
# Metadata from FITS file
if subPolicy["storage"] == "FitsStorage": # a FITS image
setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
readMetadata(location.getLocationsWithRoot()[0]))
# Add support for configuring FITS compression
addName = "add_" + datasetType
if not hasattr(self, addName):
setattr(self, addName, self.getImageCompressionSettings)
if name == "exposures":
setMethods("wcs", bypassImpl=lambda datasetType, pythonType, location, dataId:
afwGeom.makeSkyWcs(readMetadata(location.getLocationsWithRoot()[0])))
setMethods("calib", bypassImpl=lambda datasetType, pythonType, location, dataId:
afwImage.Calib(readMetadata(location.getLocationsWithRoot()[0])))
setMethods("visitInfo",
bypassImpl=lambda datasetType, pythonType, location, dataId:
afwImage.VisitInfo(readMetadata(location.getLocationsWithRoot()[0])))
setMethods("filter",
bypassImpl=lambda datasetType, pythonType, location, dataId:
afwImage.Filter(readMetadata(location.getLocationsWithRoot()[0])))
setMethods("detector",
mapImpl=lambda dataId, write=False:
dafPersist.ButlerLocation(
pythonType="lsst.afw.cameraGeom.CameraConfig",
cppType="Config",
storageName="Internal",
locationList="ignored",
dataId=dataId,
mapper=self,
storage=None,
),
bypassImpl=lambda datasetType, pythonType, location, dataId:
self.camera[self._extractDetectorName(dataId)]
)
setMethods("bbox", bypassImpl=lambda dsType, pyType, location, dataId:
afwImage.bboxFromMetadata(
readMetadata(location.getLocationsWithRoot()[0], hdu=1)))
elif name == "images":
setMethods("bbox", bypassImpl=lambda dsType, pyType, location, dataId:
afwImage.bboxFromMetadata(
readMetadata(location.getLocationsWithRoot()[0])))
if subPolicy["storage"] == "FitsCatalogStorage": # a FITS catalog
setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
readMetadata(os.path.join(location.getStorage().root,
location.getLocations()[0]), hdu=1))
# Sub-images
if subPolicy["storage"] == "FitsStorage":
def mapSubClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
subId = dataId.copy()
del subId['bbox']
loc = mapping.map(mapper, subId, write)
bbox = dataId['bbox']
llcX = bbox.getMinX()
llcY = bbox.getMinY()
width = bbox.getWidth()
height = bbox.getHeight()
loc.additionalData.set('llcX', llcX)
loc.additionalData.set('llcY', llcY)
loc.additionalData.set('width', width)
loc.additionalData.set('height', height)
if 'imageOrigin' in dataId:
loc.additionalData.set('imageOrigin',
dataId['imageOrigin'])
return loc
def querySubClosure(key, format, dataId, mapping=mapping):
subId = dataId.copy()
del subId['bbox']
return mapping.lookup(format, subId)
setMethods("sub", mapImpl=mapSubClosure, queryImpl=querySubClosure)
if subPolicy["storage"] == "FitsCatalogStorage":
# Length of catalog
setMethods("len", bypassImpl=lambda datasetType, pythonType, location, dataId:
readMetadata(os.path.join(location.getStorage().root,
location.getLocations()[0]),
hdu=1).get("NAXIS2"))
# Schema of catalog
if not datasetType.endswith("_schema") and datasetType + "_schema" not in datasets:
setMethods("schema", bypassImpl=lambda datasetType, pythonType, location, dataId:
afwTable.Schema.readFits(os.path.join(location.getStorage().root,
location.getLocations()[0])))
def map_badSourceHist(self, dataId, write):
path = "badSourceHist%(ccd)d.pickle" % dataId
return dafPersist.ButlerLocation(
"lsst.afw.image.BBox", "lsst::afw::image::BBox", "PickleStorage",
path, {}, self, dafPersist.Storage.makeFromURI(os.getcwd()))
def map_x(self, dataId, write):
path = "foo%(ccd)d.yaml" % dataId
return dafPersist.ButlerLocation(
"lsst.daf.base.PropertySet", "PropertySet", "YamlStorage", [path],
dataId, self, dafPersist.Storage.makeFromURI(self.root))
def map_x(self, dataId, write):
path = "foo%(ccd)d.pickle" % dataId
return dafPersist.ButlerLocation(
None, None, "PickleStorage", path, {}, self,
dafPersist.Storage.makeFromURI(os.getcwd()))
def map_p2(self, dataId, write):
path = "p2%(ccd)d.pickle" % dataId
path = os.path.join(self.root, self.outPath, path)
return dafPersist.ButlerLocation(
None, None, "PickleStorage", path, {}, self,
dafPersist.Storage.makeFromURI(os.getcwd()))
def testSwiftStorage(self):
"""Verify that SwiftStorage implements all the StorageInterface
functions."""
storage = SwiftStorage(uri=self.uri, create=True)
self.assertEqual(storage._containerName, self.container1Name)
self.assertTrue(storage.containerExists())
# Test containerExists by changing the container name so that it will
# return false, and then put the name back.
containerName = storage._containerName
storage._containerName = "foo"
self.assertFalse(storage.containerExists())
storage._containerName = containerName
testObject = dpTest.TestObject("abc")
butlerLocation = dp.ButlerLocation(
pythonType='lsst.daf.persistence.test.TestObject',
cppType=None,
storageName='PickleStorage',
locationList='firstTestObject',
dataId={},
mapper=None,
storage=storage)
# Test writing an object to storage
storage.write(butlerLocation, testObject)
# Test getting a local copy of the file in storage.
localFile = storage.getLocalFile('firstTestObject')
# Test reading the file in a new object using the localFile's name, as
# well as using the localFile handle directly.
for f in (open(localFile.name, 'r'), localFile):
if sys.version_info.major >= 3:
obj = pickle.load(f, encoding="latin1")
else:
obj = pickle.load(f)
self.assertEqual(testObject, obj)
# Test reading the butlerLocation, should return the object instance.
reloadedObject = storage.read(butlerLocation)
self.assertEqual(testObject, reloadedObject[0])
# Test the 'exists' function with a string
self.assertTrue(storage.exists('firstTestObject'))
self.assertFalse(storage.exists('secondTestObject'))
# Test the 'exists' function with a ButlerLocation. (note that most of
# the butler location fields are unused in exists and so are set to
# None here.)
location = dp.ButlerLocation(pythonType=None,
cppType=None,
storageName=None,
locationList=['firstTestObject'],
dataId={},
mapper=None,
storage=None)
self.assertTrue(storage.exists(location))
location = dp.ButlerLocation(pythonType=None,
cppType=None,
storageName=None,
locationList=['secondTestObject'],
dataId={},
mapper=None,
storage=None)
self.assertFalse(storage.exists(location))
# Test the 'instanceSearch' function, with and without the fits header
# extension
self.assertEqual(storage.instanceSearch('firstTestObject'),
['firstTestObject'])
self.assertEqual(storage.instanceSearch('firstTestObject[1]'),
['firstTestObject[1]'])
self.assertEqual(storage.instanceSearch('first*Object'),
['firstTestObject'])
self.assertEqual(storage.instanceSearch('*TestObject[1]'),
['firstTestObject[1]'])
self.assertIsNone(storage.instanceSearch('secondTestObject'))
self.assertIsNone(storage.instanceSearch('secondTestObject[1]'))
# Test the 'search' function
self.assertEqual(storage.search(self.uri, 'firstTestObject'),
['firstTestObject'])
# Test the copy function
storage.copyFile('firstTestObject', 'secondTestObject')
with self.assertRaises(RuntimeError):
storage.copyFile('thirdTestObject', 'fourthTestObject')
# Test locationWithRoot
self.assertEqual(storage.locationWithRoot('firstTestObject'),
self.uri + '/' + 'firstTestObject')
# Test getRepositoryCfg and putRepositoryCfg
repositoryCfg = dp.RepositoryCfg.makeFromArgs(dp.RepositoryArgs(
root=self.uri, mapper=TestMapper),
parents=None)
storage.putRepositoryCfg(repositoryCfg)
reloadedRepoCfg = storage.getRepositoryCfg(self.uri)
self.assertEqual(repositoryCfg, reloadedRepoCfg)
# Test getting a non-existant RepositoryCfg
self.assertIsNone(storage.getRepositoryCfg(self.uri2))
# Test getting the mapper class from the repoCfg in the repo.
mapper = SwiftStorage.getMapperClass(self.uri)
self.assertEqual(mapper, TestMapper)
# Test for a repoCfg that resides outside its repository; it has a
# root that is not the same as its location.
repositoryCfg = dp.RepositoryCfg.makeFromArgs(dp.RepositoryArgs(
root='foo/bar/baz', mapper='lsst.obs.base.CameraMapper'),
parents=None)
storage.putRepositoryCfg(repositoryCfg, loc=self.uri)
reloadedRepoCfg = storage.getRepositoryCfg(self.uri)
self.assertEqual(repositoryCfg, reloadedRepoCfg)
storage.deleteContainer()
self.assertFalse(storage.containerExists())
