def test(self):
"""Verify that the tags on a repository with an Old Butler repository parent are applied to that
parent
"""
# put objA in repoA:
objA = tstObj('a')
butler = dp.Butler(outputs=self.repoADir)
butler.put(objA, 'foo', {'bar': 1})
del butler
# create repoB and put objB in it:
objB = tstObj('b')
butler = dp.Butler(inputs=self.repoADir, outputs=self.repoBDir)
butler.put(objB, 'foo', {'bar': 2})
del butler
# verify that repoB can be used as an input, and objA and objB can be gotten from it:
butler = dp.Butler(inputs=self.repoBDir)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1})), objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 2})), objB)
del butler
# apply a tag and verify that repoB can still be used as an input, and both objA (in repoA) and objB
# can be gotten from it:
butler = dp.Butler(inputs={'root': self.repoBDir, 'tags': 'baz'})
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='baz')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 2}, tag='baz')),
objB)
del butler
# create a New Butler repoC and put objC in it:
objC = tstObj('c')
repoCDir = os.path.join(self.testDir, 'repoC')
butler = dp.Butler(inputs=self.repoBDir, outputs=repoCDir)
butler.put(objC, 'foo', {'bar': 3})
del butler
# verify that repoC can be used as an input, and objA, objB, and objC can be gotten from it:
butler = dp.Butler(inputs=repoCDir)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1})), objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 2})), objB)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 3})), objC)
del butler
# apply a tag and verify that repoC can be used as an input, and objA, objB, and objC can be gotten
# from it:
butler = dp.Butler(inputs={'root': repoCDir, 'tags': 'baz'})
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='baz')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 2}, tag='baz')),
objB)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 3}, tag='baz')),
objC)
del butler
def testQueryMetadata(self):
val = self.butler.queryMetadata('raw', ('filter', ))
val.sort()
self.assertEqual(val, ['g', 'r'])
val = self.butler.queryMetadata('raw', ('visit', ))
val.sort()
self.assertEqual(val, [1, 2, 3])
val = self.butler.queryMetadata('raw', ('visit', ),
dataId={'filter': 'g'})
val.sort()
self.assertEqual(val, [1, 2])
val = self.butler.queryMetadata('raw', ('visit', ),
dataId={'filter': 'r'})
self.assertEqual(val, [3])
val = self.butler.queryMetadata('raw', ('filter', ),
dataId={'visit': 1})
self.assertEqual(val, ['g'])
val = self.butler.queryMetadata('raw', ('filter', ),
dataId={'visit': 2})
self.assertEqual(val, ['g'])
val = self.butler.queryMetadata('raw', ('filter', ),
dataId={'visit': 3})
self.assertEqual(val, ['r'])
val = self.butler.queryMetadata('raw', ('visit', ),
dataId={'filter': 'h'})
self.assertEqual(val, [])
dataId = dp.DataId({'filter': 'g'}, tag='baArgs')
val = self.butler.queryMetadata('raw', ('visit', ),
dataId={'filter': 'g'})
val.sort()
self.assertEqual(val, [1, 2])
dataId = dp.DataId({'filter': 'g'}, tag='foo')
val = self.butler.queryMetadata('raw', ('visit', ), dataId=dataId)
self.assertEqual(val, [])
def __init__(self, drp_path, reducedDatasets=None):
"""The 'drp_path' input is the path to the DRP output directory."""
# Load the bulter
self.drp_path = drp_path
self.butler = dafPersist.Butler(self.drp_path)
self.mapper = self.butler._getDefaultMapper()
self.repoData = self.butler._repos.outputs()[0]
# Load some basic info on the current DRP
self.repo_input = self._get_repo(
"input") # repo containing the raw data after ingestion
self.repo_output = self._get_repo(
"output") # repo containing the processed data
# Load some dataids
self.datasetTypes = self._get_datasetTypes()
self.datasetTypes_filename = self._get_datasetTypes_withfiles()
self.dataIds = {}
defaultDatasets = [
'raw', 'forced_src', 'deepCoadd_meas', 'deepCoadd_forced_src',
'calexp'
] #, 'eimage']:
if reducedDatasets: defaultDatasets = reducedDatasets[:]
for dataset in defaultDatasets:
dataids = self.get_dataIds(dataset)
if len(dataids):
self.dataIds[dataset] = dataids
# Load filter and visits
self.filters = self.get_filter_list()
self.visits = self.get_visit_list()
# Skymap if any
if self.butler._locate('deepCoadd_skyMap',
dafPersist.DataId({}),
write=False) is not None:
self.skymap = self.butler.get("deepCoadd_skyMap")
self.skymap_name = self.skymap.__class__.__name__
self.skymap_doc = self.skymap.__doc__
self.skymap_config = self.skymap.config.toDict()
self.skymap_numtracts = self.skymap._numTracts
self.skymap_numpatches = self.skymap[0].getNumPatches()
# Mapper info
self.mapper_name = self.mapper.__name__
self.mapper_package = self.mapper.packageName
self.mapper_camera = self.mapper.getCameraName()
# Packages
self.packages = self.butler.get('packages')
# Other
self.configs = self._load_configs()
self.schemas = self._load_schemas()
def __init__(self, drp_path=None):
# Make sure we have data to load
if drp_path is None and DRPPATH is None:
raise IOError("You must give a path a DRP output directory.")
# Load the bulter
self.drp_path = drp_path if drp_path is not None else DRPPATH
self.butler = dafPersist.Butler(self.drp_path)
self.mapper = self.butler._getDefaultMapper()
self.repoData = self.butler._repos.outputs()[0]
# Load some basic info on the current DRP
self.repo_input = self._get_repo("input") # repo containing the rwe data after ingestion
self.repo_output = self._get_repo("output") # repo containing the processed data
# Load some dataids
self.datasetTypes = self._get_datasetTypes()
self.datasetTypes_filename = self._get_datasetTypes_withfiles()
self.dataIds = {}
for dataset in ['raw', 'forced_src', 'deepCoadd_meas', 'deepCoadd_forced_src']:
dataids = self.get_dataIds(dataset)
if len(dataids):
self.dataIds[dataset] = dataids
# Load filter and visits
self.filters = self.get_filter_list()
self.visits = self.get_visit_list()
# Skymap if any
if self.butler._locate('deepCoadd_skyMap', dafPersist.DataId({}), write=False) is not None:
self.skymap = self.butler.get("deepCoadd_skyMap")
self.skymap_name = self.skymap.__class__.__name__
self.skymap_doc = self.skymap.__doc__
self.skymap_config = self.skymap.config.toDict()
self.skymap_numtracts = self.skymap._numTracts
self.skymap_numpatches = self.skymap[0].getNumPatches()
# Mapper info
self.mapper_name = self.mapper.__name__
self.mapper_package = self.mapper.packageName
self.mapper_camera = self.mapper.getCameraName()
# Packages
self.packages = self.butler.get('packages')
# Other
self.configs = self._load_configs()
self.schemas = self._load_schemas()
def testOneLevelInputs(self):
"""
1. put an object with the same ID but slightly different value into 2 repositories.
2. use those repositories as inputs to a butler, and tag them
3. make sure that the correct object is gotten for each of
a. one tag
b. the other tag
c. no tag
4. repeat step 3 but reverse the order of input cfgs to a new butler.
5. use the butler from step 4 and write an output. The inputs will get recorded as parents of the
output repo.
6. create a new butler with a new overlapping repo, and verify that objects can be gotten from the
other's parent repos via tagging.
"""
objA = tstObj('a')
objB = tstObj('b')
# put objA in repo1:
repo1Args = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo1'),
mapper=MapperForTestWriting)
butler = dp.Butler(outputs=repo1Args)
butler.put(objA, 'foo', {'bar': 1})
del butler
# put objB in repo2:
repo2Args = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo2'),
mapper=MapperForTestWriting)
butler = dp.Butler(outputs=repo2Args)
butler.put(objB, 'foo', {'bar': 1})
del butler
del repo1Args
del repo2Args
# make the objects inputs of repos
# and verify the correct object can ge fetched using the tag and not using the tag
repo1Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo1'),
tags='one')
repo2Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo2'),
tags='two')
butler = dp.Butler(inputs=(repo1Args, repo2Args))
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')),
objB)
self.assertEqual(butler.get('foo', {'bar': 1}), objA)
butler = dp.Butler(inputs=(repo2Args, repo1Args))
self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='one')), objA)
self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='two')), objB)
self.assertEqual(butler.get('foo', dp.DataId(bar=1)), objB)
# create butler with repo1 and repo2 as parents, and an output repo3.
repo3Args = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo3'),
mapper=MapperForTestWriting)
butler = dp.Butler(inputs=(repo1Args, repo2Args), outputs=repo3Args)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')),
objB)
self.assertEqual(butler.get('foo', {'bar': 1}), objA)
# add an object to the output repo. note since the output repo mode is 'rw' that object is gettable
# and it has first priority in search order. Other repos should be searchable by tagging.
objC = tstObj('c')
butler.put(objC, 'foo', {'bar': 1})
self.assertEqual(butler.get('foo', {'bar': 1}), objC)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')),
objB)
del butler
repo3Cfg = dp.Storage().getRepositoryCfg(
os.path.join(self.testDir, 'repo3'))
self.assertEqual(repo3Cfg.parents, [
os.path.join(self.testDir, 'repo1'),
os.path.join(self.testDir, 'repo2')
])
# expand the structure to look like this:
# ┌────────────────────────┐ ┌────────────────────────┐
# │repo1 │ │repo2 │
# │ tag:"one" │ │ tag:"two" │
# │ tstObj('a') │ │ tstObj('b') │
# │ at ('foo', {'bar:1'})│ │ at ('foo', {'bar:1'})│
# └───────────┬────────────┘ └───────────┬────────────┘
# └─────────────┬────────────┘
# ┌────────────┴───────────┐ ┌────────────────────────┐
# │repo4 │ │repo5 │
# │ tag:"four" │ │ tag:"five" │
# │ tstObj('d') │ │ tstObj('e') │
# │ at ('foo', {'bar:2'})│ │ at ('foo', {'bar:1'})│
# └───────────┬────────────┘ └───────────┬────────────┘
# └─────────────┬────────────┘
# ┌──┴───┐
# │butler│
# └──────┘
repo4Args = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo4'),
mapper=MapperForTestWriting)
butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repo1'),
os.path.join(self.testDir, 'repo2')),
outputs=repo4Args)
objD = tstObj('d')
butler.put(objD, 'foo', {'bar': 2})
del butler
repo5Cfg = dp.RepositoryArgs(mode='rw',
root=os.path.join(self.testDir, 'repo5'),
mapper=MapperForTestWriting)
butler = dp.Butler(outputs=repo5Cfg)
objE = tstObj('e')
butler.put(objE, 'foo', {'bar': 1})
del butler
repo4Args = dp.RepositoryArgs(cfgRoot=os.path.join(
self.testDir, 'repo4'),
tags='four')
repo5Args = dp.RepositoryArgs(cfgRoot=os.path.join(
self.testDir, 'repo5'),
tags='five')
butler = dp.Butler(inputs=(repo4Args, repo5Args))
self.assertEqual(butler.get('foo', {'bar': 1}), objA)
self.assertEqual(butler.get('foo', {'bar': 2}), objD)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')),
objE)
del butler
butler = dp.Butler(inputs=(repo5Args, repo4Args))
self.assertEqual(butler.get('foo', {'bar': 1}), objE)
self.assertEqual(butler.get('foo', {'bar': 2}), objD)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')),
objA)
self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')),
objE)
del butler
