def testMultipleParentsDifferentMappers(self):
"""
creates a repo that:
a. does not have a mapper specified in the cfg
b. has 2 parent repos that have different mappers
verifies that the constructor raises a RuntimeError because the mapper can not be inferred.
"""
repoACfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoA'),
mapper=MapperForTestWriting,
mapperArgs=None,
parents=None,
policy=None)
repoBCfg = dp.RepositoryCfg(
root=os.path.join(self.testDir, 'repoB'),
mapper='lsst.daf.persistence.test.EmptyTestMapper',
mapperArgs=None,
parents=None,
policy=None)
dp.Storage.putRepositoryCfg(repoACfg,
os.path.join(self.testDir, 'repoA'))
dp.Storage.putRepositoryCfg(repoBCfg,
os.path.join(self.testDir, 'repoB'))
repoCArgs = dp.RepositoryArgs(mode='w',
root=os.path.join(self.testDir, 'repoC'))
self.assertRaises(RuntimeError,
dp.Butler,
inputs=(os.path.join(self.testDir, 'repoA'),
os.path.join(self.testDir, 'repoB')),
outputs=repoCArgs)
def testMultipleParentsSameMapper(self):
"""
creates a repo that:
a. does not have a mapper specified in the cfg
b. has 2 parents that do have the same mapper specified in the cfg
verifies that the child repo inherits the parent's mapper via inference.
"""
repoACfg = dp.RepositoryCfg(
root=os.path.join(self.testDir, 'repoA'),
mapper=MapperForTestWriting,
mapperArgs=None,
parents=None,
policy=None,
)
repoBCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoB'),
mapper=MapperForTestWriting,
mapperArgs=None,
parents=None,
policy=None)
dp.Storage.putRepositoryCfg(repoACfg,
os.path.join(self.testDir, 'repoA'))
dp.Storage.putRepositoryCfg(repoBCfg,
os.path.join(self.testDir, 'repoB'))
repoCArgs = dp.RepositoryArgs(mode='w',
root=os.path.join(self.testDir, 'repoC'))
butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoA'),
os.path.join(self.testDir, 'repoB')),
outputs=repoCArgs)
self.assertIsInstance(butler._repos.outputs()[0].repo._mapper,
MapperForTestWriting)
def test(self):
parentCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'parent'),
mapper='lsst.daf.persistence.SomeMapper',
mapperArgs={},
parents=None,
policy=None)
cfg = dp.RepositoryCfg(root=self.testDir,
mapper='lsst.daf.persistence.SomeMapper',
mapperArgs={},
parents=[parentCfg],
policy=None)
dp.PosixStorage.putRepositoryCfg(cfg)
reloadedCfg = dp.PosixStorage.getRepositoryCfg(self.testDir)
self.assertEqual(cfg, reloadedCfg)
self.assertEqual(cfg.parents[0], parentCfg)
def testReuseOutputRoot(self):
"""Set up an output repositoriy and verify its parent relationship to
the input repository.
Then set up an output repository with the first output as an input,
and verify the parent relationships.
"""
testOutput = self.mkdtemp("testOutput")
butler = dafPersist.Butler(inputs={
'root': ROOT,
'mapper': MinMapper1
},
outputs=testOutput)
self.assertTrue(os.path.exists(testOutput))
self.assertTrue(os.path.isdir(testOutput))
cfg = dafPersist.Storage().getRepositoryCfg(testOutput)
expectedCfg = dafPersist.RepositoryCfg(root=ROOT,
mapper=MinMapper1,
mapperArgs=None,
parents=None,
policy=None)
self.assertEqual(cfg.parents, [expectedCfg])
del butler
testOutput2 = self.mkdtemp("testOutput2")
butler = dafPersist.Butler(inputs={
'root': testOutput,
'mapper': MinMapper1
},
outputs=testOutput2)
self.assertTrue(os.path.exists(testOutput2))
self.assertTrue(os.path.isdir(testOutput2))
cfg = dafPersist.Storage().getRepositoryCfg(testOutput2)
self.assertEqual(cfg.parents, [testOutput])
del butler
def test(self):
cfg = dp.RepositoryCfg(root='foo',
mapper='foo',
mapperArgs=None,
parents=None,
policy=None)
self.assertIsInstance(cfg.mapperArgs, dict)
def v0Constructor(loader, node):
d = loader.construct_mapping(node)
return dp.RepositoryCfg(root=d['_root'],
mapper=None,
mapperArgs=None,
parents=None,
policy=None)
def testWriteCfg(self):
# The number of writers to use can result in too many open files
# We calculate this as the 80% of the maximum allowed number for this
# process, or 1000, whichever is smaller.
numWriters = 1000
try:
import resource
limit = resource.getrlimit(resource.RLIMIT_NOFILE)
allowedOpen = int(limit[0] * 0.8)
if allowedOpen < numWriters:
numWriters = allowedOpen
except:
# Use the default number if we had trouble obtaining resources
pass
startTime = time.time()
go = multiprocessing.Value('b', False)
cfg = dp.RepositoryCfg(root=os.path.join(self.testDir),
mapper='bar',
mapperArgs={},
parents=None,
policy=None)
procs = [
multiprocessing.Process(target=TestOneThousandWriters.writeCfg,
args=(cfg, go)) for x in range(numWriters)
]
for proc in procs:
proc.start()
go = True
for proc in procs:
proc.join()
endTime = time.time()
log = Log.getLogger("daf.persistence")
log.trace("TestOneThousandWriters took {} seconds.".format(endTime -
startTime))
def testWriteCfg(self):
"""Test parallel writes to a configuration file.
multiprocessing is used to spawn several writer function executions,
all of which wait to be released by the condition variable "go".
There are no asserts here, so success is measured solely by not
failing with an exception, but the time it took to do the writes can
be logged as a potential performance metric.
"""
numWriters = 3
startTime = time.time()
go = multiprocessing.Value('b', False)
cfg = dp.RepositoryCfg(root=os.path.join(self.testDir),
mapper='bar',
mapperArgs={},
parents=None,
policy=None)
procs = [
multiprocessing.Process(target=TestMultipleWriters.writeCfg,
args=(cfg, go)) for x in range(numWriters)
]
for proc in procs:
proc.start()
go = True
for proc in procs:
proc.join()
endTime = time.time()
log = Log.getLogger("daf.persistence")
log.trace("TestMultipleWriters took {} seconds.".format(endTime -
startTime))
def test(self):
cfg = dp.RepositoryCfg(root='.',
mapper='my.mapper.class',
mapperArgs=None,
parents=['bar', 'baz'],
policy=None)
cfg.extendParents(['bar', 'baz', 'qux'])
with self.assertRaises(dp.ParentsMismatch):
cfg.extendParents(['bar', 'baz', 'corge'])
def test(self):
butler = dp.Butler(outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'a'),
mapper=MapperForTestWriting))
del butler
os.makedirs(os.path.join(self.testDir, 'b'))
os.rename(os.path.join(self.testDir, 'a/repositoryCfg.yaml'),
os.path.join(self.testDir, 'b/repositoryCfg.yaml'))
butler = dp.Butler(inputs=os.path.join(self.testDir, 'b'))
self.assertEqual(list(butler._repos.inputs())[0].cfg,
dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'),
mapper=MapperForTestWriting,
mapperArgs=None,
parents=None,
policy=None))
def testCreateOutputRoot(self):
"""Create an input repository and a related output repo, and verify there is a parent relationship
from the output repo to the input repo."""
testOutput = self.mkdtemp("testOutput")
butler = dafPersist.Butler(inputs={
'root': ROOT,
'mapper': MinMapper1
},
outputs=testOutput)
self.assertTrue(butler)
self.assertTrue(os.path.exists(testOutput))
self.assertTrue(os.path.isdir(testOutput))
self.assertTrue(
os.path.exists(os.path.join(testOutput, "repositoryCfg.yaml")))
cfg = dafPersist.PosixStorage.getRepositoryCfg(testOutput)
expectedCfg = dafPersist.RepositoryCfg(root=ROOT,
mapper=MinMapper1,
mapperArgs=None,
parents=None,
policy=None)
self.assertEqual(len(cfg.parents), 1)
self.assertEqual(cfg.parents[0], expectedCfg)
def test(self):
# create a repo where repo 'a' is a parent of repo 'b'
butler = dp.Butler(outputs=dp.RepositoryArgs(
root=os.path.join(self.testDir, 'a'), mapper=MapperForTestWriting))
del butler
butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
outputs=os.path.join(self.testDir, 'b'))
self.assertEqual(len(butler._repos.inputs()), 1)
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
del butler
# load that repo a few times, include 'a' as an input.
for i in range(4):
butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
outputs=dp.RepositoryArgs(root=os.path.join(
self.testDir, 'b'),
mode='rw'))
self.assertEqual(len(butler._repos.inputs()), 2)
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
self.assertEqual(butler._repos.inputs()[1].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
cfg = dp.Storage().getRepositoryCfg(os.path.join(
self.testDir, 'b'))
self.assertEqual(
cfg,
dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'),
mapper=MapperForTestWriting,
mapperArgs=None,
parents=[os.path.join(self.testDir, 'a')],
policy=None))
# load the repo a few times and don't explicitly list 'a' as an input
for i in range(4):
butler = dp.Butler(outputs=dp.RepositoryArgs(
root=os.path.join(self.testDir, 'b'), mode='rw'))
self.assertEqual(len(butler._repos.inputs()), 2)
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
self.assertEqual(butler._repos.inputs()[1].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
cfg = dp.Storage().getRepositoryCfg(os.path.join(
self.testDir, 'b'))
self.assertEqual(
cfg,
dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'),
mapper=MapperForTestWriting,
mapperArgs=None,
parents=[os.path.join(self.testDir, 'a')],
policy=None))
# load 'b' as 'write only' and don't list 'a' as an input. This should raise, because inputs must
# match readable outputs parents.
with self.assertRaises(RuntimeError):
butler = dp.Butler(outputs=os.path.join(self.testDir, 'b'))
# load 'b' as 'write only' and explicitly list 'a' as an input.
butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
outputs=os.path.join(self.testDir, 'b'))
self.assertEqual(len(butler._repos.inputs()), 1)
self.assertEqual(len(butler._repos.outputs()), 1)
self.assertEqual(butler._repos.inputs()[0].cfg.root,
os.path.join(self.testDir, 'a'))
self.assertEqual(butler._repos.outputs()[0].cfg.root,
os.path.join(self.testDir, 'b'))
cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b'))
