def testReuseOutputRoot(self): """Set up an output repositoriy and verify its parent relationship to the input repository. Then set up an output repository with the first output as an input, and verify the parent relationships. """ testOutput = self.mkdtemp("testOutput") butler = dafPersist.Butler(inputs={ 'root': ROOT, 'mapper': MinMapper1 }, outputs=testOutput) self.assertTrue(os.path.exists(testOutput)) self.assertTrue(os.path.isdir(testOutput)) cfg = dafPersist.Storage().getRepositoryCfg(testOutput) expectedCfg = dafPersist.RepositoryCfg(root=ROOT, mapper=MinMapper1, mapperArgs=None, parents=None, policy=None) self.assertEqual(cfg.parents, [expectedCfg]) del butler testOutput2 = self.mkdtemp("testOutput2") butler = dafPersist.Butler(inputs={ 'root': testOutput, 'mapper': MinMapper1 }, outputs=testOutput2) self.assertTrue(os.path.exists(testOutput2)) self.assertTrue(os.path.isdir(testOutput2)) cfg = dafPersist.Storage().getRepositoryCfg(testOutput2) self.assertEqual(cfg.parents, [testOutput]) del butler
def testStorageRepoCfgCache(self): """Tests that when a cfg is gotten from storage it is cached.""" butler = dp.Butler( outputs=dp.RepositoryArgs(mode='w', mapper=dpTest.EmptyTestMapper, root=os.path.join(self.testDir, 'a'))) del butler storage = dp.Storage() self.assertEqual(0, len(storage.repositoryCfgs)) cfg = storage.getRepositoryCfg(os.path.join(self.testDir, 'a')) self.assertEqual( cfg, storage.repositoryCfgs[os.path.join(self.testDir, 'a')])
def test(self): # create a repo where repo 'a' is a parent of repo 'b' butler = dp.Butler(outputs=dp.RepositoryArgs( root=os.path.join(self.testDir, 'a'), mapper=MapperForTestWriting)) del butler butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), outputs=os.path.join(self.testDir, 'b')) self.assertEqual(len(butler._repos.inputs()), 1) self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'a')) self.assertEqual(len(butler._repos.outputs()), 1) self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) del butler # load that repo a few times, include 'a' as an input. for i in range(4): butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), outputs=dp.RepositoryArgs(root=os.path.join( self.testDir, 'b'), mode='rw')) self.assertEqual(len(butler._repos.inputs()), 2) self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'b')) self.assertEqual(butler._repos.inputs()[1].cfg.root, os.path.join(self.testDir, 'a')) self.assertEqual(len(butler._repos.outputs()), 1) self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) cfg = dp.Storage().getRepositoryCfg(os.path.join( self.testDir, 'b')) self.assertEqual( cfg, dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'), mapper=MapperForTestWriting, mapperArgs=None, parents=[os.path.join(self.testDir, 'a')], policy=None)) # load the repo a few times and don't explicitly list 'a' as an input for i in range(4): butler = dp.Butler(outputs=dp.RepositoryArgs( root=os.path.join(self.testDir, 'b'), mode='rw')) self.assertEqual(len(butler._repos.inputs()), 2) self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'b')) self.assertEqual(butler._repos.inputs()[1].cfg.root, os.path.join(self.testDir, 'a')) self.assertEqual(len(butler._repos.outputs()), 1) self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) cfg = dp.Storage().getRepositoryCfg(os.path.join( self.testDir, 'b')) self.assertEqual( cfg, dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'), mapper=MapperForTestWriting, mapperArgs=None, parents=[os.path.join(self.testDir, 'a')], policy=None)) # load 'b' as 'write only' and don't list 'a' as an input. This should raise, because inputs must # match readable outputs parents. with self.assertRaises(RuntimeError): butler = dp.Butler(outputs=os.path.join(self.testDir, 'b')) # load 'b' as 'write only' and explicitly list 'a' as an input. butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), outputs=os.path.join(self.testDir, 'b')) self.assertEqual(len(butler._repos.inputs()), 1) self.assertEqual(len(butler._repos.outputs()), 1) self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'a')) self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b'))
def testOneLevelInputs(self): """ 1. put an object with the same ID but slightly different value into 2 repositories. 2. use those repositories as inputs to a butler, and tag them 3. make sure that the correct object is gotten for each of a. one tag b. the other tag c. no tag 4. repeat step 3 but reverse the order of input cfgs to a new butler. 5. use the butler from step 4 and write an output. The inputs will get recorded as parents of the output repo. 6. create a new butler with a new overlapping repo, and verify that objects can be gotten from the other's parent repos via tagging. """ objA = tstObj('a') objB = tstObj('b') # put objA in repo1: repo1Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo1'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repo1Args) butler.put(objA, 'foo', {'bar': 1}) del butler # put objB in repo2: repo2Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo2'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repo2Args) butler.put(objB, 'foo', {'bar': 1}) del butler del repo1Args del repo2Args # make the objects inputs of repos # and verify the correct object can ge fetched using the tag and not using the tag repo1Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo1'), tags='one') repo2Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo2'), tags='two') butler = dp.Butler(inputs=(repo1Args, repo2Args)) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) self.assertEqual(butler.get('foo', {'bar': 1}), objA) butler = dp.Butler(inputs=(repo2Args, repo1Args)) self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='two')), objB) self.assertEqual(butler.get('foo', dp.DataId(bar=1)), objB) # create butler with repo1 and repo2 as parents, and an output repo3. repo3Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo3'), mapper=MapperForTestWriting) butler = dp.Butler(inputs=(repo1Args, repo2Args), outputs=repo3Args) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) self.assertEqual(butler.get('foo', {'bar': 1}), objA) # add an object to the output repo. note since the output repo mode is 'rw' that object is gettable # and it has first priority in search order. Other repos should be searchable by tagging. objC = tstObj('c') butler.put(objC, 'foo', {'bar': 1}) self.assertEqual(butler.get('foo', {'bar': 1}), objC) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) del butler repo3Cfg = dp.Storage().getRepositoryCfg( os.path.join(self.testDir, 'repo3')) self.assertEqual(repo3Cfg.parents, [ os.path.join(self.testDir, 'repo1'), os.path.join(self.testDir, 'repo2') ]) # expand the structure to look like this: # ┌────────────────────────┐ ┌────────────────────────┐ # │repo1 │ │repo2 │ # │ tag:"one" │ │ tag:"two" │ # │ tstObj('a') │ │ tstObj('b') │ # │ at ('foo', {'bar:1'})│ │ at ('foo', {'bar:1'})│ # └───────────┬────────────┘ └───────────┬────────────┘ # └─────────────┬────────────┘ # ┌────────────┴───────────┐ ┌────────────────────────┐ # │repo4 │ │repo5 │ # │ tag:"four" │ │ tag:"five" │ # │ tstObj('d') │ │ tstObj('e') │ # │ at ('foo', {'bar:2'})│ │ at ('foo', {'bar:1'})│ # └───────────┬────────────┘ └───────────┬────────────┘ # └─────────────┬────────────┘ # ┌──┴───┐ # │butler│ # └──────┘ repo4Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo4'), mapper=MapperForTestWriting) butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repo1'), os.path.join(self.testDir, 'repo2')), outputs=repo4Args) objD = tstObj('d') butler.put(objD, 'foo', {'bar': 2}) del butler repo5Cfg = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo5'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repo5Cfg) objE = tstObj('e') butler.put(objE, 'foo', {'bar': 1}) del butler repo4Args = dp.RepositoryArgs(cfgRoot=os.path.join( self.testDir, 'repo4'), tags='four') repo5Args = dp.RepositoryArgs(cfgRoot=os.path.join( self.testDir, 'repo5'), tags='five') butler = dp.Butler(inputs=(repo4Args, repo5Args)) self.assertEqual(butler.get('foo', {'bar': 1}), objA) self.assertEqual(butler.get('foo', {'bar': 2}), objD) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE) del butler butler = dp.Butler(inputs=(repo5Args, repo4Args)) self.assertEqual(butler.get('foo', {'bar': 1}), objE) self.assertEqual(butler.get('foo', {'bar': 2}), objD) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE) del butler