def test(self):
# create a repo where repo 'a' is a parent of repo 'b'
butler = dp.Butler(outputs=dp.RepositoryArgs(
root=os.path.join(self.testDir, 'a'), mapper=MapperForTestWriting))
del butler
butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
outputs=os.path.join(self.testDir, 'b'))
self.assertEqual(len(butler._repos.inputs()), 1)
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
del butler
# load that repo a few times, include 'a' as an input.
for i in range(4):
butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
outputs=dp.RepositoryArgs(root=os.path.join(
self.testDir, 'b'),
mode='rw'))
self.assertEqual(len(butler._repos.inputs()), 2)
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
self.assertEqual(butler._repos.inputs()[1].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
cfg = dp.Storage().getRepositoryCfg(os.path.join(
self.testDir, 'b'))
self.assertEqual(
cfg,
dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'),
mapper=MapperForTestWriting,
mapperArgs=None,
parents=[os.path.join(self.testDir, 'a')],
policy=None))
# load the repo a few times and don't explicitly list 'a' as an input
for i in range(4):
butler = dp.Butler(outputs=dp.RepositoryArgs(
root=os.path.join(self.testDir, 'b'), mode='rw'))
self.assertEqual(len(butler._repos.inputs()), 2)
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
self.assertEqual(butler._repos.inputs()[1].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
cfg = dp.Storage().getRepositoryCfg(os.path.join(
self.testDir, 'b'))
self.assertEqual(
cfg,
dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'),
mapper=MapperForTestWriting,
mapperArgs=None,
parents=[os.path.join(self.testDir, 'a')],
policy=None))
# load 'b' as 'write only' and don't list 'a' as an input. This should raise, because inputs must
# match readable outputs parents.
with self.assertRaises(RuntimeError):
butler = dp.Butler(outputs=os.path.join(self.testDir, 'b'))
# load 'b' as 'write only' and explicitly list 'a' as an input.
butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
outputs=os.path.join(self.testDir, 'b'))
self.assertEqual(len(butler._repos.inputs()), 1)
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b'))
def testOneLevelInputs(self):
"""
1. put an object with the same ID but slightly different value into 2 repositories.
2. use those repositories as inputs to a butler, and tag them
3. make sure that the correct object is gotten for each of
a. one tag
b. the other tag
c. no tag
4. repeat step 3 but reverse the order of input cfgs to a new butler.
5. use the butler from step 4 and write an output. The inputs will get recorded as parents of the
output repo.
6. create a new butler with a new overlapping repo, and verify that objects can be gotten from the
other's parent repos via tagging.
"""
objA = tstObj('a')
objB = tstObj('b')
# put objA in repo1:
repo1Args = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo1'),
mapper=MapperForTestWriting)
butler = dp.Butler(outputs=repo1Args)
butler.put(objA, 'foo', {'bar': 1})
del butler
# put objB in repo2:
repo2Args = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo2'),
mapper=MapperForTestWriting)
butler = dp.Butler(outputs=repo2Args)
butler.put(objB, 'foo', {'bar': 1})
del butler
del repo1Args
del repo2Args
# make the objects inputs of repos
# and verify the correct object can ge fetched using the tag and not using the tag
repo1Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo1'),
tags='one')
repo2Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo2'),
tags='two')
butler = dp.Butler(inputs=(repo1Args, repo2Args))
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')),
objB)
self.assertEqual(butler.get('foo', {'bar': 1}), objA)
butler = dp.Butler(inputs=(repo2Args, repo1Args))
self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='one')), objA)
self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='two')), objB)
self.assertEqual(butler.get('foo', dp.DataId(bar=1)), objB)
# create butler with repo1 and repo2 as parents, and an output repo3.
repo3Args = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo3'),
mapper=MapperForTestWriting)
butler = dp.Butler(inputs=(repo1Args, repo2Args), outputs=repo3Args)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')),
objB)
self.assertEqual(butler.get('foo', {'bar': 1}), objA)
# add an object to the output repo. note since the output repo mode is 'rw' that object is gettable
# and it has first priority in search order. Other repos should be searchable by tagging.
objC = tstObj('c')
butler.put(objC, 'foo', {'bar': 1})
self.assertEqual(butler.get('foo', {'bar': 1}), objC)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')),
objB)
del butler
repo3Cfg = dp.Storage().getRepositoryCfg(
os.path.join(self.testDir, 'repo3'))
self.assertEqual(repo3Cfg.parents, [
os.path.join(self.testDir, 'repo1'),
os.path.join(self.testDir, 'repo2')
])
# expand the structure to look like this:
# ┌────────────────────────┐ ┌────────────────────────┐
# │repo1 │ │repo2 │
# │ tag:"one" │ │ tag:"two" │
# │ tstObj('a') │ │ tstObj('b') │
# │ at ('foo', {'bar:1'})│ │ at ('foo', {'bar:1'})│
# └───────────┬────────────┘ └───────────┬────────────┘
# └─────────────┬────────────┘
# ┌────────────┴───────────┐ ┌────────────────────────┐
# │repo4 │ │repo5 │
# │ tag:"four" │ │ tag:"five" │
# │ tstObj('d') │ │ tstObj('e') │
# │ at ('foo', {'bar:2'})│ │ at ('foo', {'bar:1'})│
# └───────────┬────────────┘ └───────────┬────────────┘
# └─────────────┬────────────┘
# ┌──┴───┐
# │butler│
# └──────┘
repo4Args = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo4'),
mapper=MapperForTestWriting)
butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repo1'),
os.path.join(self.testDir, 'repo2')),
outputs=repo4Args)
objD = tstObj('d')
butler.put(objD, 'foo', {'bar': 2})
del butler
repo5Cfg = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo5'),
mapper=MapperForTestWriting)
butler = dp.Butler(outputs=repo5Cfg)
objE = tstObj('e')
butler.put(objE, 'foo', {'bar': 1})
del butler
repo4Args = dp.RepositoryArgs(cfgRoot=os.path.join(
self.testDir, 'repo4'),
tags='four')
repo5Args = dp.RepositoryArgs(cfgRoot=os.path.join(
self.testDir, 'repo5'),
tags='five')
butler = dp.Butler(inputs=(repo4Args, repo5Args))
self.assertEqual(butler.get('foo', {'bar': 1}), objA)
self.assertEqual(butler.get('foo', {'bar': 2}), objD)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')),
objE)
del butler
butler = dp.Butler(inputs=(repo5Args, repo4Args))
self.assertEqual(butler.get('foo', {'bar': 1}), objE)
self.assertEqual(butler.get('foo', {'bar': 2}), objD)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')),
objE)
del butler
def testExistingParents(self):
# parents of inputs should be added to the inputs list
butler = dp.Butler(
outputs=dp.RepositoryArgs(mode='w',
mapper=dpTest.EmptyTestMapper,
root=os.path.join(self.testDir, 'a')))
del butler
butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
outputs=os.path.join(self.testDir, 'b'))
del butler
butler = dp.Butler(inputs=os.path.join(self.testDir, 'b'))
self.assertEqual(len(butler._repos.inputs()), 2)
# verify serach order:
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
self.assertEqual(butler._repos.inputs()[1].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(len(butler._repos.outputs()), 0)
butler = dp.Butler(outputs=dp.RepositoryArgs(
cfgRoot=os.path.join(self.testDir, 'b'), mode='rw'))
# verify serach order:
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
self.assertEqual(butler._repos.inputs()[1].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
outputs=dp.RepositoryArgs(cfgRoot=os.path.join(
self.testDir, 'b'),
mode='rw'))
self.assertEqual(len(butler._repos.inputs()), 2)
# verify serach order:
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
self.assertEqual(butler._repos.inputs()[1].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
# parents of write-only outputs must be be listed with the inputs
with self.assertRaises(RuntimeError):
butler = dp.Butler(outputs=os.path.join(self.testDir, 'b'))
butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
outputs=os.path.join(self.testDir, 'b'))
self.assertEqual(len(butler._repos.inputs()), 1)
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
# add a new parent to an existing output
butler = dp.Butler(
outputs=dp.RepositoryArgs(mode='w',
mapper=dpTest.EmptyTestMapper,
root=os.path.join(self.testDir, 'c')))
butler = dp.Butler(inputs=(os.path.join(self.testDir, 'a'),
os.path.join(self.testDir, 'c')),
outputs=os.path.join(self.testDir, 'b'))
# should raise if the input order gets reversed:
with self.assertRaises(RuntimeError):
butler = dp.Butler(inputs=(os.path.join(self.testDir, 'c'),
os.path.join(self.testDir, 'a')),
outputs=os.path.join(self.testDir, 'b'))
def setUp(self):
self.testData = tempfile.mkdtemp(dir=ROOT,
prefix='TestCompositeTestCase-')
self.firstRepoPath = os.path.join(self.testData, 'repo1')
self.objA = dpTest.TestObject("abc")
self.objB = dpTest.TestObject("def")
self.policy = dafPersist.Policy({
'camera': 'lsst.afw.cameraGeom.Camera',
'datasets': {
'basicObject1': {
'python': 'lsst.daf.persistence.test.TestObject',
'template': 'basic/id%(id)s.pickle',
'storage': 'PickleStorage'
},
'basicObject2': {
'python': 'lsst.daf.persistence.test.TestObject',
'template': 'basic/name%(name)s.pickle',
'storage': 'PickleStorage'
},
'basicPair': {
'python':
'lsst.daf.persistence.test.TestObjectPair',
'composite': {
'a': {
'datasetType': 'basicObject1'
},
'b': {
'datasetType': 'basicObject2'
}
},
'assembler':
'lsst.daf.persistence.test.TestObjectPair.assembler',
'disassembler':
'lsst.daf.persistence.test.TestObjectPair.disassembler'
},
'stdTestType': {
'python': 'lsst.daf.persistence.test.TestObjectPair',
'composite': {
'a': {
'datasetType': 'basicObject1'
},
'b': {
'datasetType': 'basicObject2'
}
}
},
'bypassTestType': {
'python': 'lsst.daf.persistence.test.TestObjectPair',
'composite': {
'a': {
'datasetType': 'basicObject1'
},
'b': {
'datasetType': 'basicObject2'
}
}
}
}
})
repoArgs = dafPersist.RepositoryArgs(
root=self.firstRepoPath,
policy=self.policy,
mapper='lsst.obs.base.test.CompositeMapper')
butler = dafPersist.Butler(outputs=repoArgs)
butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
del butler
del repoArgs
def setUp(self):
self.testData = tempfile.mkdtemp(dir=ROOT,
prefix='TestGenericAssembler-')
self.firstRepoPath = os.path.join(self.testData, 'repo1')
self.secondRepoPath = os.path.join(self.testData, 'repo2')
self.objA = dpTest.TestObject("abc")
self.objB = dpTest.TestObject("def")
self.policy = dafPersist.Policy({
'camera': 'lsst.afw.cameraGeom.Camera',
'datasets': {
'basicObject1': {
'python': 'lsst.daf.persistence.test.TestObject',
'template': 'basic/id%(id)s.pickle',
'storage': 'PickleStorage'
},
'basicObject2': {
'python': 'lsst.daf.persistence.test.TestObject',
'template': 'basic/name%(name)s.pickle',
'storage': 'PickleStorage'
},
'basicPair': {
'python': 'lsst.daf.persistence.test.TestObjectPair',
'composite': {
'a': {
'datasetType': 'basicObject1'
},
'b': {
'datasetType': 'basicObject2'
}
},
# note, no assembler or disassembler specified here, will use
# setter names inferred by component name.
},
# "generic assembler default constructor pair"
'gaDefCtorPair':
{ # dataset defition that uses the default ctor
'python': 'lsst.daf.persistence.test.TestObjectPair',
'composite': {
# note that the component names are the same as the argument
# names in the TestObjectPair.__init__ func.
'objA': {
'datasetType': 'basicObject1',
'getter': 'get_a'
},
'objB': {
'datasetType': 'basicObject2',
'getter': 'get_b'
}
},
# note, no assembler or disassembler specified here.
},
# "generic assembler default "
'gaPairWithSetter': {
'python': 'lsst.daf.persistence.test.TestObjectPair',
'composite': {
# note that the component names do not match argument names
# in the TestObjectPair.__init__ func or the set functions
# in the python object.
'z': {
'datasetType': 'basicObject1',
'setter': 'set_a',
'getter': 'get_a'
},
'x': {
'datasetType': 'basicObject2',
'setter': 'set_b',
'getter': 'get_b'
}
}
},
# simple object where setter and getter is named with underscore
# separator
'underscoreSetter': {
'python':
'lsst.daf.persistence.test.TestObjectUnderscoreSetter',
'composite': {
'foo': {
'datasetType': 'basicObject1'
}
}
},
# simple object where setter and getter is named with camelcase
'camelCaseSetter': {
'python':
'lsst.daf.persistence.test.TestObjectCamelCaseSetter',
'composite': {
'foo': {
'datasetType': 'basicObject1'
}
}
}
}
})
repoArgs = dafPersist.RepositoryArgs(
root=self.firstRepoPath,
policy=self.policy,
mapper='lsst.obs.base.test.CompositeMapper')
butler = dafPersist.Butler(outputs=repoArgs)
butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
del butler
del repoArgs
def testSwiftStorage(self):
"""Verify that SwiftStorage implements all the StorageInterface
functions."""
storage = SwiftStorage(uri=self.uri, create=True)
self.assertEqual(storage._containerName, self.container1Name)
self.assertTrue(storage.containerExists())
# Test containerExists by changing the container name so that it will
# return false, and then put the name back.
containerName = storage._containerName
storage._containerName = "foo"
self.assertFalse(storage.containerExists())
storage._containerName = containerName
testObject = dpTest.TestObject("abc")
butlerLocation = dp.ButlerLocation(
pythonType='lsst.daf.persistence.test.TestObject',
cppType=None,
storageName='PickleStorage',
locationList='firstTestObject',
dataId={},
mapper=None,
storage=storage)
# Test writing an object to storage
storage.write(butlerLocation, testObject)
# Test getting a local copy of the file in storage.
localFile = storage.getLocalFile('firstTestObject')
# Test reading the file in a new object using the localFile's name, as
# well as using the localFile handle directly.
for f in (open(localFile.name, 'r'), localFile):
if sys.version_info.major >= 3:
obj = pickle.load(f, encoding="latin1")
else:
obj = pickle.load(f)
self.assertEqual(testObject, obj)
# Test reading the butlerLocation, should return the object instance.
reloadedObject = storage.read(butlerLocation)
self.assertEqual(testObject, reloadedObject[0])
# Test the 'exists' function with a string
self.assertTrue(storage.exists('firstTestObject'))
self.assertFalse(storage.exists('secondTestObject'))
# Test the 'exists' function with a ButlerLocation. (note that most of
# the butler location fields are unused in exists and so are set to
# None here.)
location = dp.ButlerLocation(pythonType=None,
cppType=None,
storageName=None,
locationList=['firstTestObject'],
dataId={},
mapper=None,
storage=None)
self.assertTrue(storage.exists(location))
location = dp.ButlerLocation(pythonType=None,
cppType=None,
storageName=None,
locationList=['secondTestObject'],
dataId={},
mapper=None,
storage=None)
self.assertFalse(storage.exists(location))
# Test the 'instanceSearch' function, with and without the fits header
# extension
self.assertEqual(storage.instanceSearch('firstTestObject'),
['firstTestObject'])
self.assertEqual(storage.instanceSearch('firstTestObject[1]'),
['firstTestObject[1]'])
self.assertEqual(storage.instanceSearch('first*Object'),
['firstTestObject'])
self.assertEqual(storage.instanceSearch('*TestObject[1]'),
['firstTestObject[1]'])
self.assertIsNone(storage.instanceSearch('secondTestObject'))
self.assertIsNone(storage.instanceSearch('secondTestObject[1]'))
# Test the 'search' function
self.assertEqual(storage.search(self.uri, 'firstTestObject'),
['firstTestObject'])
# Test the copy function
storage.copyFile('firstTestObject', 'secondTestObject')
with self.assertRaises(RuntimeError):
storage.copyFile('thirdTestObject', 'fourthTestObject')
# Test locationWithRoot
self.assertEqual(storage.locationWithRoot('firstTestObject'),
self.uri + '/' + 'firstTestObject')
# Test getRepositoryCfg and putRepositoryCfg
repositoryCfg = dp.RepositoryCfg.makeFromArgs(dp.RepositoryArgs(
root=self.uri, mapper=TestMapper),
parents=None)
storage.putRepositoryCfg(repositoryCfg)
reloadedRepoCfg = storage.getRepositoryCfg(self.uri)
self.assertEqual(repositoryCfg, reloadedRepoCfg)
# Test getting a non-existant RepositoryCfg
self.assertIsNone(storage.getRepositoryCfg(self.uri2))
# Test getting the mapper class from the repoCfg in the repo.
mapper = SwiftStorage.getMapperClass(self.uri)
self.assertEqual(mapper, TestMapper)
# Test for a repoCfg that resides outside its repository; it has a
# root that is not the same as its location.
repositoryCfg = dp.RepositoryCfg.makeFromArgs(dp.RepositoryArgs(
root='foo/bar/baz', mapper='lsst.obs.base.CameraMapper'),
parents=None)
storage.putRepositoryCfg(repositoryCfg, loc=self.uri)
reloadedRepoCfg = storage.getRepositoryCfg(self.uri)
self.assertEqual(repositoryCfg, reloadedRepoCfg)
storage.deleteContainer()
self.assertFalse(storage.containerExists())
