def testMultipleParentsDifferentMappers(self): """ creates a repo that: a. does not have a mapper specified in the cfg b. has 2 parent repos that have different mappers verifies that the constructor raises a RuntimeError because the mapper can not be inferred. """ repoACfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoA'), mapper=MapperForTestWriting, mapperArgs=None, parents=None, policy=None) repoBCfg = dp.RepositoryCfg( root=os.path.join(self.testDir, 'repoB'), mapper='lsst.daf.persistence.test.EmptyTestMapper', mapperArgs=None, parents=None, policy=None) dp.Storage.putRepositoryCfg(repoACfg, os.path.join(self.testDir, 'repoA')) dp.Storage.putRepositoryCfg(repoBCfg, os.path.join(self.testDir, 'repoB')) repoCArgs = dp.RepositoryArgs(mode='w', root=os.path.join(self.testDir, 'repoC')) self.assertRaises(RuntimeError, dp.Butler, inputs=(os.path.join(self.testDir, 'repoA'), os.path.join(self.testDir, 'repoB')), outputs=repoCArgs)
def testMultipleParentsSameMapper(self): """ creates a repo that: a. does not have a mapper specified in the cfg b. has 2 parents that do have the same mapper specified in the cfg verifies that the child repo inherits the parent's mapper via inference. """ repoACfg = dp.RepositoryCfg( root=os.path.join(self.testDir, 'repoA'), mapper=MapperForTestWriting, mapperArgs=None, parents=None, policy=None, ) repoBCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoB'), mapper=MapperForTestWriting, mapperArgs=None, parents=None, policy=None) dp.Storage.putRepositoryCfg(repoACfg, os.path.join(self.testDir, 'repoA')) dp.Storage.putRepositoryCfg(repoBCfg, os.path.join(self.testDir, 'repoB')) repoCArgs = dp.RepositoryArgs(mode='w', root=os.path.join(self.testDir, 'repoC')) butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoA'), os.path.join(self.testDir, 'repoB')), outputs=repoCArgs) self.assertIsInstance(butler._repos.outputs()[0].repo._mapper, MapperForTestWriting)
def test(self): parentCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'parent'), mapper='lsst.daf.persistence.SomeMapper', mapperArgs={}, parents=None, policy=None) cfg = dp.RepositoryCfg(root=self.testDir, mapper='lsst.daf.persistence.SomeMapper', mapperArgs={}, parents=[parentCfg], policy=None) dp.PosixStorage.putRepositoryCfg(cfg) reloadedCfg = dp.PosixStorage.getRepositoryCfg(self.testDir) self.assertEqual(cfg, reloadedCfg) self.assertEqual(cfg.parents[0], parentCfg)
def testReuseOutputRoot(self): """Set up an output repositoriy and verify its parent relationship to the input repository. Then set up an output repository with the first output as an input, and verify the parent relationships. """ testOutput = self.mkdtemp("testOutput") butler = dafPersist.Butler(inputs={ 'root': ROOT, 'mapper': MinMapper1 }, outputs=testOutput) self.assertTrue(os.path.exists(testOutput)) self.assertTrue(os.path.isdir(testOutput)) cfg = dafPersist.Storage().getRepositoryCfg(testOutput) expectedCfg = dafPersist.RepositoryCfg(root=ROOT, mapper=MinMapper1, mapperArgs=None, parents=None, policy=None) self.assertEqual(cfg.parents, [expectedCfg]) del butler testOutput2 = self.mkdtemp("testOutput2") butler = dafPersist.Butler(inputs={ 'root': testOutput, 'mapper': MinMapper1 }, outputs=testOutput2) self.assertTrue(os.path.exists(testOutput2)) self.assertTrue(os.path.isdir(testOutput2)) cfg = dafPersist.Storage().getRepositoryCfg(testOutput2) self.assertEqual(cfg.parents, [testOutput]) del butler
def test(self): cfg = dp.RepositoryCfg(root='foo', mapper='foo', mapperArgs=None, parents=None, policy=None) self.assertIsInstance(cfg.mapperArgs, dict)
def v0Constructor(loader, node): d = loader.construct_mapping(node) return dp.RepositoryCfg(root=d['_root'], mapper=None, mapperArgs=None, parents=None, policy=None)
def testWriteCfg(self): # The number of writers to use can result in too many open files # We calculate this as the 80% of the maximum allowed number for this # process, or 1000, whichever is smaller. numWriters = 1000 try: import resource limit = resource.getrlimit(resource.RLIMIT_NOFILE) allowedOpen = int(limit[0] * 0.8) if allowedOpen < numWriters: numWriters = allowedOpen except: # Use the default number if we had trouble obtaining resources pass startTime = time.time() go = multiprocessing.Value('b', False) cfg = dp.RepositoryCfg(root=os.path.join(self.testDir), mapper='bar', mapperArgs={}, parents=None, policy=None) procs = [ multiprocessing.Process(target=TestOneThousandWriters.writeCfg, args=(cfg, go)) for x in range(numWriters) ] for proc in procs: proc.start() go = True for proc in procs: proc.join() endTime = time.time() log = Log.getLogger("daf.persistence") log.trace("TestOneThousandWriters took {} seconds.".format(endTime - startTime))
def testWriteCfg(self): """Test parallel writes to a configuration file. multiprocessing is used to spawn several writer function executions, all of which wait to be released by the condition variable "go". There are no asserts here, so success is measured solely by not failing with an exception, but the time it took to do the writes can be logged as a potential performance metric. """ numWriters = 3 startTime = time.time() go = multiprocessing.Value('b', False) cfg = dp.RepositoryCfg(root=os.path.join(self.testDir), mapper='bar', mapperArgs={}, parents=None, policy=None) procs = [ multiprocessing.Process(target=TestMultipleWriters.writeCfg, args=(cfg, go)) for x in range(numWriters) ] for proc in procs: proc.start() go = True for proc in procs: proc.join() endTime = time.time() log = Log.getLogger("daf.persistence") log.trace("TestMultipleWriters took {} seconds.".format(endTime - startTime))
def test(self): cfg = dp.RepositoryCfg(root='.', mapper='my.mapper.class', mapperArgs=None, parents=['bar', 'baz'], policy=None) cfg.extendParents(['bar', 'baz', 'qux']) with self.assertRaises(dp.ParentsMismatch): cfg.extendParents(['bar', 'baz', 'corge'])
def test(self): butler = dp.Butler(outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'a'), mapper=MapperForTestWriting)) del butler os.makedirs(os.path.join(self.testDir, 'b')) os.rename(os.path.join(self.testDir, 'a/repositoryCfg.yaml'), os.path.join(self.testDir, 'b/repositoryCfg.yaml')) butler = dp.Butler(inputs=os.path.join(self.testDir, 'b')) self.assertEqual(list(butler._repos.inputs())[0].cfg, dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'), mapper=MapperForTestWriting, mapperArgs=None, parents=None, policy=None))
def testCreateOutputRoot(self): """Create an input repository and a related output repo, and verify there is a parent relationship from the output repo to the input repo.""" testOutput = self.mkdtemp("testOutput") butler = dafPersist.Butler(inputs={ 'root': ROOT, 'mapper': MinMapper1 }, outputs=testOutput) self.assertTrue(butler) self.assertTrue(os.path.exists(testOutput)) self.assertTrue(os.path.isdir(testOutput)) self.assertTrue( os.path.exists(os.path.join(testOutput, "repositoryCfg.yaml"))) cfg = dafPersist.PosixStorage.getRepositoryCfg(testOutput) expectedCfg = dafPersist.RepositoryCfg(root=ROOT, mapper=MinMapper1, mapperArgs=None, parents=None, policy=None) self.assertEqual(len(cfg.parents), 1) self.assertEqual(cfg.parents[0], expectedCfg)
def test(self): # create a repo where repo 'a' is a parent of repo 'b' butler = dp.Butler(outputs=dp.RepositoryArgs( root=os.path.join(self.testDir, 'a'), mapper=MapperForTestWriting)) del butler butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), outputs=os.path.join(self.testDir, 'b')) self.assertEqual(len(butler._repos.inputs()), 1) self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'a')) self.assertEqual(len(butler._repos.outputs()), 1) self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) del butler # load that repo a few times, include 'a' as an input. for i in range(4): butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), outputs=dp.RepositoryArgs(root=os.path.join( self.testDir, 'b'), mode='rw')) self.assertEqual(len(butler._repos.inputs()), 2) self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'b')) self.assertEqual(butler._repos.inputs()[1].cfg.root, os.path.join(self.testDir, 'a')) self.assertEqual(len(butler._repos.outputs()), 1) self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) cfg = dp.Storage().getRepositoryCfg(os.path.join( self.testDir, 'b')) self.assertEqual( cfg, dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'), mapper=MapperForTestWriting, mapperArgs=None, parents=[os.path.join(self.testDir, 'a')], policy=None)) # load the repo a few times and don't explicitly list 'a' as an input for i in range(4): butler = dp.Butler(outputs=dp.RepositoryArgs( root=os.path.join(self.testDir, 'b'), mode='rw')) self.assertEqual(len(butler._repos.inputs()), 2) self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'b')) self.assertEqual(butler._repos.inputs()[1].cfg.root, os.path.join(self.testDir, 'a')) self.assertEqual(len(butler._repos.outputs()), 1) self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) cfg = dp.Storage().getRepositoryCfg(os.path.join( self.testDir, 'b')) self.assertEqual( cfg, dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'), mapper=MapperForTestWriting, mapperArgs=None, parents=[os.path.join(self.testDir, 'a')], policy=None)) # load 'b' as 'write only' and don't list 'a' as an input. This should raise, because inputs must # match readable outputs parents. with self.assertRaises(RuntimeError): butler = dp.Butler(outputs=os.path.join(self.testDir, 'b')) # load 'b' as 'write only' and explicitly list 'a' as an input. butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), outputs=os.path.join(self.testDir, 'b')) self.assertEqual(len(butler._repos.inputs()), 1) self.assertEqual(len(butler._repos.outputs()), 1) self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'a')) self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b'))