def write(self, butlerLocation, obj):
"""Writes an object to a location and persistence format specified by
ButlerLocation
This file uses PosixStorage to write the object to a file on disk (that
is to say: serialize it). Then the file is uploaded to the swift
container. When we have better support for pluggable serializers,
hopefully the first step of writing to disk can be skipped and the
object can be serialzied and streamed directly to the swift container.
Parameters
----------
butlerLocation : ButlerLocation
The location & formatting for the object to be written.
obj : object instance
The object to be written.
"""
swiftLocations = butlerLocation.getLocations()
# Here the ButlerLocation is modified sligtly to write to a temporary
# file via PosixStorage. (Then the temporary file is written to the
# swift container)
localFile = tempfile.NamedTemporaryFile()
butlerLocation.locationList = [localFile.name]
butlerLocation.storage = dafPersist.PosixStorage('/', create=False)
butlerLocation.storage.write(butlerLocation, obj)
butlerLocation.locationList = swiftLocations
self._putFile(swiftLocations[0], localFile.name)
def read(self, butlerLocation):
"""Read from a butlerLocation.
This implementation downloads the file object from the swift container
to a temporary file and instantiates the object using
PosixStorage.read. The temporary file will be deleted when the handle
to temporary file is deleted, so the handle is monkey patched onto the
local file to keep the file from being deleted as long as the object
exists.
Parameters
----------
butlerLocation : ButlerLocation
The location & formatting for the object(s) to be read.
Returns
-------
A list of objects as described by the butler location. One item for
each location in butlerLocation.getLocations()
"""
localFile = self.getLocalFile(butlerLocation.getLocations()[0])
butlerLocation.locationList = [localFile.name]
butlerLocation.storage = dafPersist.PosixStorage('/', create=False)
obj = butlerLocation.storage.read(butlerLocation)
return obj
def testFilePathIn2ndParentParent(self):
"""Find a file in a repo that is the parent of a parent of the root repo."""
grandParentDir = os.path.join(self.testDir, 'a')
parentDir = os.path.join(self.testDir, 'b')
childDir = os.path.join(self.testDir, 'c')
for d in (grandParentDir, parentDir, childDir):
os.makedirs(d)
for name in ('foo.txt', 'bar.txt'):
with open(os.path.join(grandParentDir, name), 'w') as f:
f.write('abc')
os.symlink('../a', os.path.join(parentDir, '_parent'))
os.symlink('../b', os.path.join(childDir, '_parent'))
storage = dafPersist.PosixStorage(uri=childDir,
create=True)
for name in ('foo.txt', 'bar.txt[0]'):
foundName = storage.search(storage.root, name, searchParents=True)
self.assertEqual(storage.root, childDir)
self.assertEqual(foundName, [os.path.join('_parent/_parent/', name)])
for name in ('foo.txt', 'bar.txt[0]'):
searchFor = os.path.join(childDir, name)
foundName = storage.search(storage.root, searchFor, searchParents=True)
self.assertEqual(storage.root, childDir)
self.assertEqual(foundName, [os.path.join(childDir, '_parent/_parent/', name)])
def testFilePathWithHeaderExt(self):
"""Find a file with a search string that includes a FITS-style header extension."""
with open(os.path.join(self.testDir, 'foo.txt'), 'w') as f:
f.write('abc')
storage = dafPersist.PosixStorage(uri=self.testDir,
create=True)
foundName = storage.search(storage.root, 'foo.txt[0]', searchParents=True)
self.assertEqual(foundName, ['foo.txt[0]'])
searchFor = os.path.join(self.testDir, 'foo.txt[0]')
foundName = storage.search(storage.root, searchFor, searchParents=True)
self.assertEqual(foundName, [searchFor])
def testDoSearchParentFlag(self):
"""Test that parent search can be told to follow _parent symlink (or not) when searching."""
parentDir = os.path.join(self.testDir, 'a')
childDir = os.path.join(self.testDir, 'b')
for d in (parentDir, childDir):
os.makedirs(d)
with open(os.path.join(parentDir, 'foo.txt'), 'w') as f:
f.write('abc')
os.symlink('../a', os.path.join(childDir, '_parent'))
storage = dafPersist.PosixStorage(uri=childDir, create=True)
self.assertEqual(storage.search(storage.root, 'foo.txt', searchParents=True), ['_parent/foo.txt'])
self.assertEqual(storage.search(storage.root, 'foo.txt', searchParents=False), None)
def testAbsolutePath(self):
"""Tests that GetLocalFile returns a file when it exists and returns
None when it does not exist."""
storage = dp.PosixStorage(self.testDir, create=True)
self.assertIsNone(storage.getLocalFile('foo.txt'))
with open(os.path.join(self.testDir, 'foo.txt'), 'w') as f:
f.write('foobarbaz')
del f
f = storage.getLocalFile('foo.txt')
self.assertIsInstance(f, FileType)
self.assertEqual(f.read(), 'foobarbaz')
f.close()
def testFilePath(self):
"""Test that a file can be found; when root is part of the path then root is returned with the path
result. When root is not part of the path then root is not returned with the path result."""
with open(os.path.join(self.testDir, 'foo.txt'), 'w') as f:
f.write('abc')
storage = dafPersist.PosixStorage(uri=self.testDir,
create=True)
foundName = storage.search(storage.root, 'foo.txt', searchParents=True)
self.assertEqual(foundName, ['foo.txt'])
searchFor = os.path.join(self.testDir, 'foo.txt')
foundName = storage.search(storage.root, searchFor, searchParents=True)
self.assertEqual(foundName, [searchFor])
def testFilePathInParent(self):
"""Find a file in a repo that is a grandchild of the repo that has the file"""
parentDir = os.path.join(self.testDir, 'a')
childDir = os.path.join(self.testDir, 'b')
for d in (parentDir, childDir):
os.makedirs(d)
with open(os.path.join(parentDir, 'foo.txt'), 'w') as f:
f.write('abc')
os.symlink('../a', os.path.join(childDir, '_parent'))
storage = dafPersist.PosixStorage(uri=childDir,
create=True)
foundName = storage.search(storage.root, 'foo.txt', searchParents=True)
self.assertEqual(storage.root, childDir)
self.assertEqual(foundName, ['_parent/foo.txt'])
searchFor = os.path.join(childDir, 'foo.txt')
foundName = storage.search(storage.root, searchFor, searchParents=True)
self.assertEqual(storage.root, childDir)
self.assertEqual(foundName, [os.path.join(childDir, '_parent/foo.txt')])
def testNoResults(self):
storage = dafPersist.PosixStorage(uri=self.testDir, create=True)
self.assertIsNone(storage.search(storage.root, 'fileThatDoesNotExist.txt', searchParents=True))