def test(self): objAbc = tstObj('abc') objDef = tstObj('def') objGhi = tstObj('ghi') objJkl = tstObj('jkl') repoAArgs = dp.RepositoryArgs(mode='w', root=os.path.join(self.testDir, 'repoA'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repoAArgs) butler.put(objAbc, 'foo', {'bar': 1}) butler.put(objDef, 'foo', {'bar': 2}) repoBArgs = dp.RepositoryArgs(mode='w', root=os.path.join(self.testDir, 'repoB'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repoBArgs) butler.put(objGhi, 'foo', {'bar': 1}) # note different object with overlapping dataId with repoA butler.put(objJkl, 'foo', {'bar': 3}) # note repo order: A, B butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoA'), os.path.join(self.testDir, 'repoB'))) self.assertEqual(butler.get('foo', {'bar': 1}), objAbc) self.assertEqual(butler.get('foo', {'bar': 2}), objDef) self.assertEqual(butler.get('foo', {'bar': 3}), objJkl) # note reversed repo order: B, A butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoB'), os.path.join(self.testDir, 'repoA'))) self.assertEqual(butler.get('foo', {'bar': 1}), objGhi) self.assertEqual(butler.get('foo', {'bar': 2}), objDef) self.assertEqual(butler.get('foo', {'bar': 3}), objJkl)
def testCreateAggregateAndLoadingAChild(self): """Tests putting a very basic pickled object in a variety of Repository configuration settings :return: """ repoAArgs = dp.RepositoryArgs(mode='w', root=os.path.join(self.testDir, 'repoA'), mapper=MapperForTestWriting) repoBArgs = dp.RepositoryArgs(mode='w', root=os.path.join(self.testDir, 'repoB'), mapper=MapperForTestWriting) butlerAB = dp.Butler(outputs=[repoAArgs, repoBArgs]) objA = tstObj('abc') butlerAB.put(objA, 'foo', {'val': 1}) objB = tstObj('def') butlerAB.put(objB, 'foo', {'val': 2}) # create butlers where the output repos are now input repos butlerC = dp.Butler(inputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoA'))) butlerD = dp.Butler(inputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoB'))) # # verify the objects exist by getting them self.assertEqual(objA, butlerC.get('foo', {'val': 1})) self.assertEqual(objA, butlerC.get('foo', {'val': 1})) self.assertEqual(objB, butlerD.get('foo', {'val': 2})) self.assertEqual(objB, butlerD.get('foo', {'val': 2}))
def testCreateAggregateAndLoadingAChild(self): """Tests putting a very basic pickled object in a variety of Repository configuration settings :return: """ repoAArgs = dp.RepositoryArgs(mode='w', root=os.path.join(self.testDir, 'repoA'), mapper=MapperForTestWriting) repoBArgs = dp.RepositoryArgs(mode='w', root=os.path.join(self.testDir, 'repoB'), mapper=MapperForTestWriting) butlerAB = dp.Butler(outputs=[repoAArgs, repoBArgs]) objA = tstObj('abc') butlerAB.put(objA, 'foo', {'val': 1}) objB = tstObj('def') butlerAB.put(objB, 'foo', {'val': 2}) # create butlers where the output repos are now input repos butlerC = dp.Butler(inputs=dp.RepositoryArgs( root=os.path.join(self.testDir, 'repoA'))) butlerD = dp.Butler(inputs=dp.RepositoryArgs( root=os.path.join(self.testDir, 'repoB'))) # # verify the objects exist by getting them self.assertEqual(objA, butlerC.get('foo', {'val': 1})) self.assertEqual(objA, butlerC.get('foo', {'val': 1})) self.assertEqual(objB, butlerD.get('foo', {'val': 2})) self.assertEqual(objB, butlerD.get('foo', {'val': 2}))
def test(self): """Verify that the tags on a repository with an Old Butler repository parent are applied to that parent """ # put objA in repoA: objA = tstObj('a') butler = dp.Butler(outputs=self.repoADir) butler.put(objA, 'foo', {'bar': 1}) del butler # create repoB and put objB in it: objB = tstObj('b') butler = dp.Butler(inputs=self.repoADir, outputs=self.repoBDir) butler.put(objB, 'foo', {'bar': 2}) del butler # verify that repoB can be used as an input, and objA and objB can be gotten from it: butler = dp.Butler(inputs=self.repoBDir) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1})), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 2})), objB) del butler # apply a tag and verify that repoB can still be used as an input, and both objA (in repoA) and objB # can be gotten from it: butler = dp.Butler(inputs={'root': self.repoBDir, 'tags': 'baz'}) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='baz')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 2}, tag='baz')), objB) del butler # create a New Butler repoC and put objC in it: objC = tstObj('c') repoCDir = os.path.join(self.testDir, 'repoC') butler = dp.Butler(inputs=self.repoBDir, outputs=repoCDir) butler.put(objC, 'foo', {'bar': 3}) del butler # verify that repoC can be used as an input, and objA, objB, and objC can be gotten from it: butler = dp.Butler(inputs=repoCDir) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1})), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 2})), objB) self.assertEqual(butler.get('foo', dp.DataId({'bar': 3})), objC) del butler # apply a tag and verify that repoC can be used as an input, and objA, objB, and objC can be gotten # from it: butler = dp.Butler(inputs={'root': repoCDir, 'tags': 'baz'}) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='baz')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 2}, tag='baz')), objB) self.assertEqual(butler.get('foo', dp.DataId({'bar': 3}, tag='baz')), objC) del butler
def testPutWithIncompleteDataId(self): repoAArgs = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repoA'), mapper=NoResultsMapper) butler = dp.Butler(outputs=repoAArgs) obj = tstObj('abc') with self.assertRaises(RuntimeError): butler.put(obj, 'list', {'visit': '2'}) with self.assertRaises(RuntimeError): butler.put(obj, 'None', {'visit': '2'})
def test(self): repoAArgs = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoA'), mapper=MapperForTestWriting) repoBArgs = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoB'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=(repoAArgs, repoBArgs)) obj0 = tstObj('abc') butler.put(obj0, 'foo', {'bar': 1}) for root in (os.path.join(self.testDir, 'repoA'), os.path.join(self.testDir, 'repoB')): butler = dp.Butler(inputs=root) self.assertEqual(butler.get('foo', {'bar': 1}), obj0)
def test(self): repoAArgs = dp.RepositoryArgs(mode='w', root=os.path.join(self.testDir, 'repoA'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repoAArgs) obj0 = tstObj('abc') butler.put(obj0, 'foo', {'bar': 1}) del butler del repoAArgs repoBArgs = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repoB'), mapper=MapperForTestWriting) butler = dp.Butler(inputs=os.path.join(self.testDir, 'repoA'), outputs=repoBArgs) obj1 = butler.get('foo', {'bar': 1}) self.assertEqual(obj0, obj1) obj1.data = "def" butler.put(obj1, 'foo', {'bar': 1}) obj2 = butler.get('foo', {'bar': 1}) self.assertEqual(obj1, obj2)
def testOneLevelInputs(self): """ 1. put an object with the same ID but slightly different value into 2 repositories. 2. use those repositories as inputs to a butler, and tag them 3. make sure that the correct object is gotten for each of a. one tag b. the other tag c. no tag 4. repeat step 3 but reverse the order of input cfgs to a new butler. 5. use the butler from step 4 and write an output. The inputs will get recorded as parents of the output repo. 6. create a new butler with a new overlapping repo, and verify that objects can be gotten from the other's parent repos via tagging. """ objA = tstObj('a') objB = tstObj('b') # put objA in repo1: repo1Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo1'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repo1Args) butler.put(objA, 'foo', {'bar': 1}) del butler # put objB in repo2: repo2Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo2'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repo2Args) butler.put(objB, 'foo', {'bar': 1}) del butler del repo1Args del repo2Args # make the objects inputs of repos # and verify the correct object can ge fetched using the tag and not using the tag repo1Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo1'), tags='one') repo2Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo2'), tags='two') butler = dp.Butler(inputs=(repo1Args, repo2Args)) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) self.assertEqual(butler.get('foo', {'bar': 1}), objA) butler = dp.Butler(inputs=(repo2Args, repo1Args)) self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='two')), objB) self.assertEqual(butler.get('foo', dp.DataId(bar=1)), objB) # create butler with repo1 and repo2 as parents, and an output repo3. repo3Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo3'), mapper=MapperForTestWriting) butler = dp.Butler(inputs=(repo1Args, repo2Args), outputs=repo3Args) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) self.assertEqual(butler.get('foo', {'bar': 1}), objA) # add an object to the output repo. note since the output repo mode is 'rw' that object is gettable # and it has first priority in search order. Other repos should be searchable by tagging. objC = tstObj('c') butler.put(objC, 'foo', {'bar': 1}) self.assertEqual(butler.get('foo', {'bar': 1}), objC) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) del butler repo3Cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'repo3')) self.assertEqual(repo3Cfg.parents, [os.path.join(self.testDir, 'repo1'), os.path.join(self.testDir, 'repo2')]) # expand the structure to look like this: # ┌────────────────────────┐ ┌────────────────────────┐ # │repo1 │ │repo2 │ # │ tag:"one" │ │ tag:"two" │ # │ tstObj('a') │ │ tstObj('b') │ # │ at ('foo', {'bar:1'})│ │ at ('foo', {'bar:1'})│ # └───────────┬────────────┘ └───────────┬────────────┘ # └─────────────┬────────────┘ # ┌────────────┴───────────┐ ┌────────────────────────┐ # │repo4 │ │repo5 │ # │ tag:"four" │ │ tag:"five" │ # │ tstObj('d') │ │ tstObj('e') │ # │ at ('foo', {'bar:2'})│ │ at ('foo', {'bar:1'})│ # └───────────┬────────────┘ └───────────┬────────────┘ # └─────────────┬────────────┘ # ┌──┴───┐ # │butler│ # └──────┘ repo4Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo4'), mapper=MapperForTestWriting) butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repo1'), os.path.join(self.testDir, 'repo2')), outputs=repo4Args) objD = tstObj('d') butler.put(objD, 'foo', {'bar': 2}) del butler repo5Cfg = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo5'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repo5Cfg) objE = tstObj('e') butler.put(objE, 'foo', {'bar': 1}) del butler repo4Args = dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'repo4'), tags='four') repo5Args = dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'repo5'), tags='five') butler = dp.Butler(inputs=(repo4Args, repo5Args)) self.assertEqual(butler.get('foo', {'bar': 1}), objA) self.assertEqual(butler.get('foo', {'bar': 2}), objD) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE) del butler butler = dp.Butler(inputs=(repo5Args, repo4Args)) self.assertEqual(butler.get('foo', {'bar': 1}), objE) self.assertEqual(butler.get('foo', {'bar': 2}), objD) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE) del butler
def testOneLevelInputs(self): """ 1. put an object with the same ID but slightly different value into 2 repositories. 2. use those repositories as inputs to a butler, and tag them 3. make sure that the correct object is gotten for each of a. one tag b. the other tag c. no tag 4. repeat step 3 but reverse the order of input cfgs to a new butler. 5. use the butler from step 4 and write an output. The inputs will get recorded as parents of the output repo. 6. create a new butler with a new overlapping repo, and verify that objects can be gotten from the other's parent repos via tagging. """ objA = tstObj('a') objB = tstObj('b') # put objA in repo1: repo1Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo1'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repo1Args) butler.put(objA, 'foo', {'bar': 1}) del butler # put objB in repo2: repo2Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo2'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repo2Args) butler.put(objB, 'foo', {'bar': 1}) del butler del repo1Args del repo2Args # make the objects inputs of repos # and verify the correct object can ge fetched using the tag and not using the tag repo1Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo1'), tags='one') repo2Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo2'), tags='two') butler = dp.Butler(inputs=(repo1Args, repo2Args)) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) self.assertEqual(butler.get('foo', {'bar': 1}), objA) butler = dp.Butler(inputs=(repo2Args, repo1Args)) self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='two')), objB) self.assertEqual(butler.get('foo', dp.DataId(bar=1)), objB) # create butler with repo1 and repo2 as parents, and an output repo3. repo3Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo3'), mapper=MapperForTestWriting) butler = dp.Butler(inputs=(repo1Args, repo2Args), outputs=repo3Args) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) self.assertEqual(butler.get('foo', {'bar': 1}), objA) # add an object to the output repo. note since the output repo mode is 'rw' that object is gettable # and it has first priority in search order. Other repos should be searchable by tagging. objC = tstObj('c') butler.put(objC, 'foo', {'bar': 1}) self.assertEqual(butler.get('foo', {'bar': 1}), objC) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) del butler repo3Cfg = dp.Storage().getRepositoryCfg( os.path.join(self.testDir, 'repo3')) self.assertEqual(repo3Cfg.parents, [ os.path.join(self.testDir, 'repo1'), os.path.join(self.testDir, 'repo2') ]) # expand the structure to look like this: # ┌────────────────────────┐ ┌────────────────────────┐ # │repo1 │ │repo2 │ # │ tag:"one" │ │ tag:"two" │ # │ tstObj('a') │ │ tstObj('b') │ # │ at ('foo', {'bar:1'})│ │ at ('foo', {'bar:1'})│ # └───────────┬────────────┘ └───────────┬────────────┘ # └─────────────┬────────────┘ # ┌────────────┴───────────┐ ┌────────────────────────┐ # │repo4 │ │repo5 │ # │ tag:"four" │ │ tag:"five" │ # │ tstObj('d') │ │ tstObj('e') │ # │ at ('foo', {'bar:2'})│ │ at ('foo', {'bar:1'})│ # └───────────┬────────────┘ └───────────┬────────────┘ # └─────────────┬────────────┘ # ┌──┴───┐ # │butler│ # └──────┘ repo4Args = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo4'), mapper=MapperForTestWriting) butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repo1'), os.path.join(self.testDir, 'repo2')), outputs=repo4Args) objD = tstObj('d') butler.put(objD, 'foo', {'bar': 2}) del butler repo5Cfg = dp.RepositoryArgs(mode='rw', root=os.path.join(self.testDir, 'repo5'), mapper=MapperForTestWriting) butler = dp.Butler(outputs=repo5Cfg) objE = tstObj('e') butler.put(objE, 'foo', {'bar': 1}) del butler repo4Args = dp.RepositoryArgs(cfgRoot=os.path.join( self.testDir, 'repo4'), tags='four') repo5Args = dp.RepositoryArgs(cfgRoot=os.path.join( self.testDir, 'repo5'), tags='five') butler = dp.Butler(inputs=(repo4Args, repo5Args)) self.assertEqual(butler.get('foo', {'bar': 1}), objA) self.assertEqual(butler.get('foo', {'bar': 2}), objD) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE) del butler butler = dp.Butler(inputs=(repo5Args, repo4Args)) self.assertEqual(butler.get('foo', {'bar': 1}), objE) self.assertEqual(butler.get('foo', {'bar': 2}), objD) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA) self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE) del butler