def test_missing_storage(self):
# When a file exists in the DB but is missing from disk, a 404
# is just confusing. It's an internal error, so 500 instead.
client = LibrarianClient()
# Upload a file so we can retrieve it.
sample_data = b'blah'
file_alias_id = client.addFile('sample',
len(sample_data),
BytesIO(sample_data),
contentType='text/plain')
url = client.getURLForAlias(file_alias_id)
# Change the date_created to a known value that doesn't match
# the disk timestamp. The timestamp on disk cannot be trusted.
file_alias = IMasterStore(LibraryFileAlias).get(
LibraryFileAlias, file_alias_id)
# Commit so the file is available from the Librarian.
self.commit()
# Fetch the file via HTTP.
response = requests.get(url)
response.raise_for_status()
# Delete the on-disk file.
storage = LibrarianStorage(config.librarian_server.root, None)
os.remove(storage._fileLocation(file_alias.contentID))
# The URL now 500s, since the DB says it should exist.
response = requests.get(url)
self.assertEqual(500, response.status_code)
self.assertIn('Server', response.headers)
self.assertNotIn('Last-Modified', response.headers)
self.assertNotIn('Cache-Control', response.headers)
def setUp(self):
self.directory = tempfile.mkdtemp()
self.storage = LibrarianStorage(self.directory, db.Library())
# Hook the commit and rollback methods of the store.
self.store = IStore(LibraryFileContent)
self.committed = self.rolledback = False
self.orig_commit = self.store.commit
self.orig_rollback = self.store.rollback
def commit():
self.committed = True
self.orig_commit()
self.store.commit = commit
def rollback():
self.rolledback = True
self.orig_rollback()
self.store.rollback = rollback
def setUp(self):
self.directory = tempfile.mkdtemp()
self.storage = LibrarianStorage(self.directory, db.Library())
# Hook the commit and rollback methods of the store.
self.store = IStore(LibraryFileContent)
self.committed = self.rolledback = False
self.orig_commit = self.store.commit
self.orig_rollback = self.store.rollback
def commit():
self.committed = True
self.orig_commit()
self.store.commit = commit
def rollback():
self.rolledback = True
self.orig_rollback()
self.store.rollback = rollback
def setUp(self):
switch_dbuser('librarian')
self.directory = tempfile.mkdtemp()
self.storage = LibrarianStorage(self.directory, db.Library())
class LibrarianStorageDBTests(unittest.TestCase):
layer = LaunchpadZopelessLayer
def setUp(self):
switch_dbuser('librarian')
self.directory = tempfile.mkdtemp()
self.storage = LibrarianStorage(self.directory, db.Library())
def tearDown(self):
shutil.rmtree(self.directory, ignore_errors=True)
def test_addFile(self):
data = 'data ' * 50
digest = hashlib.sha1(data).hexdigest()
newfile = self.storage.startAddFile('file1', len(data))
newfile.srcDigest = digest
newfile.append(data)
fileid, aliasid = newfile.store()
self.failUnless(self.storage.hasFile(fileid))
def test_addFiles_identical(self):
# Start adding two files with identical data
data = 'data ' * 5000
newfile1 = self.storage.startAddFile('file1', len(data))
newfile2 = self.storage.startAddFile('file2', len(data))
newfile1.append(data)
newfile2.append(data)
id1, alias1 = newfile1.store()
id2, alias2 = newfile2.store()
# Make sure we actually got an id
self.assertNotEqual(None, id1)
self.assertNotEqual(None, id2)
# But they are two different ids, because we leave duplicate handling
# to the garbage collector
self.failIfEqual(id1, id2)
def test_badDigest(self):
data = 'data ' * 50
digest = 'crud'
newfile = self.storage.startAddFile('file', len(data))
newfile.srcDigest = digest
newfile.append(data)
self.assertRaises(DigestMismatchError, newfile.store)
def test_alias(self):
# Add a file (and so also add an alias)
data = 'data ' * 50
newfile = self.storage.startAddFile('file1', len(data))
newfile.mimetype = 'text/unknown'
newfile.append(data)
fileid, aliasid = newfile.store()
# Check that its alias has the right mimetype
fa = self.storage.getFileAlias(aliasid, None, '/')
self.assertEqual('text/unknown', fa.mimetype)
# Re-add the same file, with the same name and mimetype...
newfile2 = self.storage.startAddFile('file1', len(data))
newfile2.mimetype = 'text/unknown'
newfile2.append(data)
fileid2, aliasid2 = newfile2.store()
# Verify that we didn't get back the same alias ID
self.assertNotEqual(fa.id,
self.storage.getFileAlias(aliasid2, None, '/').id)
def test_clientProvidedDuplicateIDs(self):
# This test checks the new behaviour specified by LibrarianTransactions
# spec: don't create IDs in DB, but do check they don't exist.
# Create a new file
newfile = LibraryFileUpload(self.storage, 'filename', 0)
# Set a content ID on the file (same as would happen with a
# client-generated ID) and store it
newfile.contentID = 666
newfile.store()
newfile = LibraryFileUpload(self.storage, 'filename', 0)
newfile.contentID = 666
self.assertRaises(DuplicateFileIDError, newfile.store)
def test_clientProvidedDuplicateContent(self):
# Check the new behaviour specified by LibrarianTransactions
# spec: allow duplicate content with distinct IDs.
content = 'some content'
# Store a file with id 6661
newfile1 = LibraryFileUpload(self.storage, 'filename', 0)
newfile1.contentID = 6661
newfile1.append(content)
fileid1, aliasid1 = newfile1.store()
# Store second file identical to the first, with id 6662
newfile2 = LibraryFileUpload(self.storage, 'filename', 0)
newfile2.contentID = 6662
newfile2.append(content)
fileid2, aliasid2 = newfile2.store()
# Create rows in the database for these files.
LibraryFileContent(
filesize=0, sha1='foo', md5='xx', sha256='xx', id=6661)
LibraryFileContent(
filesize=0, sha1='foo', md5='xx', sha256='xx', id=6662)
flush_database_updates()
class LibrarianStorageTestCase(unittest.TestCase):
"""Librarian test cases that don't involve the database"""
layer = LaunchpadZopelessLayer
def setUp(self):
self.directory = tempfile.mkdtemp()
self.storage = LibrarianStorage(self.directory, db.Library())
# Hook the commit and rollback methods of the store.
self.store = IStore(LibraryFileContent)
self.committed = self.rolledback = False
self.orig_commit = self.store.commit
self.orig_rollback = self.store.rollback
def commit():
self.committed = True
self.orig_commit()
self.store.commit = commit
def rollback():
self.rolledback = True
self.orig_rollback()
self.store.rollback = rollback
def tearDown(self):
shutil.rmtree(self.directory, ignore_errors=True)
del self.store.commit
del self.store.rollback
self.orig_commit = self.orig_rollback = None
def test_hasFile_missing(self):
# Make sure hasFile returns False when a file is missing
self.failIf(self.storage.hasFile(9999999))
def _sameFileTestHelper(self, data1, data2):
# Make two temporary files
fd1, path1 = tempfile.mkstemp()
fd2, path2 = tempfile.mkstemp()
file1 = os.fdopen(fd1, 'wb')
file2 = os.fdopen(fd2, 'wb')
# Put the test data in them, and close them
file1.write(data1)
file2.write(data2)
file1.close()
file2.close()
# Do the test, and clean up afterwards
try:
return _sameFile(path1, path2)
finally:
os.remove(path1)
os.remove(path2)
def test_sameFile(self):
# Make sure sameFile returns True when the files are the same
self.failUnless(
self._sameFileTestHelper('data ' * 5000, 'data ' * 5000))
def test_notSameFile(self):
# Make sure sameFile returns False when the files are different, even
# if they are the same length.
self.failIf(self._sameFileTestHelper('data ' * 5000, 'fred ' * 5000))
def test_differentFileShorter(self):
# Make sure sameFile returns False when the second file is shorter
# than the first, even if they were the same up to that length.
self.failIf(self._sameFileTestHelper('data ' * 5000, 'data ' * 4999))
def test_differentFileLonger(self):
# Make sure sameFile returns False when the second file is longer than
# the first, even if they were the same up to that length.
self.failIf(self._sameFileTestHelper('data ' * 5000, 'data ' * 5001))
def test_prefixDirectories(self):
# _relFileLocation splits eight hex digits across four path segments
self.assertEqual('12/34/56/78', _relFileLocation(0x12345678))
# less than eight hex digits will be padded
self.assertEqual('00/00/01/23', _relFileLocation(0x123))
# more than eight digits will make the final segment longer, if that
# were to ever happen
self.assertEqual('12/34/56/789', _relFileLocation(0x123456789))
def test_multipleFilesInOnePrefixedDirectory(self):
# Check that creating a file that will be saved in 11/11/11/11
# followed by a file that will be saved in 11/11/11/12 works
# correctly -- i.e that creating a file works both if the directory
# already exists, and if the directory doesn't already exist.
self.storage.library = StubLibrary()
data = 'data ' * 50
newfile = self.storage.startAddFile('file', len(data))
newfile.contentID = 0x11111111
newfile.append(data)
fileid1, aliasid = newfile.store()
# First id from stub library should be 0x11111111
self.assertEqual(0x11111111, fileid1)
data += 'more data'
newfile = self.storage.startAddFile('file', len(data))
newfile.contentID = 0x11111112
newfile.append(data)
fileid2, aliasid = newfile.store()
# Second id from stub library should be 0x11111112
self.assertEqual(0x11111112, fileid2)
# Did the files both get stored?
self.failUnless(self.storage.hasFile(fileid1))
self.failUnless(self.storage.hasFile(fileid2))
def test_hashes(self):
# Check that the MD5, SHA1 and SHA256 hashes are correct.
data = 'i am some data'
md5 = hashlib.md5(data).hexdigest()
sha1 = hashlib.sha1(data).hexdigest()
sha256 = hashlib.sha256(data).hexdigest()
newfile = self.storage.startAddFile('file', len(data))
newfile.append(data)
lfc_id, lfa_id = newfile.store()
lfc = LibraryFileContent.get(lfc_id)
self.assertEqual(md5, lfc.md5)
self.assertEqual(sha1, lfc.sha1)
self.assertEqual(sha256, lfc.sha256)
class LibrarianStorageTestCase(unittest.TestCase):
"""Librarian test cases that don't involve the database"""
layer = LaunchpadZopelessLayer
def setUp(self):
self.directory = tempfile.mkdtemp()
self.storage = LibrarianStorage(self.directory, db.Library())
# Hook the commit and rollback methods of the store.
self.store = IStore(LibraryFileContent)
self.committed = self.rolledback = False
self.orig_commit = self.store.commit
self.orig_rollback = self.store.rollback
def commit():
self.committed = True
self.orig_commit()
self.store.commit = commit
def rollback():
self.rolledback = True
self.orig_rollback()
self.store.rollback = rollback
def tearDown(self):
shutil.rmtree(self.directory, ignore_errors=True)
del self.store.commit
del self.store.rollback
self.orig_commit = self.orig_rollback = None
def test_hasFile_missing(self):
# Make sure hasFile returns False when a file is missing
self.failIf(self.storage.hasFile(9999999))
def _sameFileTestHelper(self, data1, data2):
# Make two temporary files
fd1, path1 = tempfile.mkstemp()
fd2, path2 = tempfile.mkstemp()
file1 = os.fdopen(fd1, 'wb')
file2 = os.fdopen(fd2, 'wb')
# Put the test data in them, and close them
file1.write(data1)
file2.write(data2)
file1.close()
file2.close()
# Do the test, and clean up afterwards
try:
return _sameFile(path1, path2)
finally:
os.remove(path1)
os.remove(path2)
def test_sameFile(self):
# Make sure sameFile returns True when the files are the same
self.failUnless(self._sameFileTestHelper('data ' * 5000,
'data ' * 5000))
def test_notSameFile(self):
# Make sure sameFile returns False when the files are different, even
# if they are the same length.
self.failIf(self._sameFileTestHelper('data ' * 5000, 'fred ' * 5000))
def test_differentFileShorter(self):
# Make sure sameFile returns False when the second file is shorter
# than the first, even if they were the same up to that length.
self.failIf(self._sameFileTestHelper('data ' * 5000, 'data ' * 4999))
def test_differentFileLonger(self):
# Make sure sameFile returns False when the second file is longer than
# the first, even if they were the same up to that length.
self.failIf(self._sameFileTestHelper('data ' * 5000, 'data ' * 5001))
def test_prefixDirectories(self):
# _relFileLocation splits eight hex digits across four path segments
self.assertEqual('12/34/56/78', _relFileLocation(0x12345678))
# less than eight hex digits will be padded
self.assertEqual('00/00/01/23', _relFileLocation(0x123))
# more than eight digits will make the final segment longer, if that
# were to ever happen
self.assertEqual('12/34/56/789', _relFileLocation(0x123456789))
def test_multipleFilesInOnePrefixedDirectory(self):
# Check that creating a file that will be saved in 11/11/11/11
# followed by a file that will be saved in 11/11/11/12 works
# correctly -- i.e that creating a file works both if the directory
# already exists, and if the directory doesn't already exist.
self.storage.library = StubLibrary()
data = 'data ' * 50
newfile = self.storage.startAddFile('file', len(data))
newfile.contentID = 0x11111111
newfile.append(data)
fileid1, aliasid = newfile.store()
# First id from stub library should be 0x11111111
self.assertEqual(0x11111111, fileid1)
data += 'more data'
newfile = self.storage.startAddFile('file', len(data))
newfile.contentID = 0x11111112
newfile.append(data)
fileid2, aliasid = newfile.store()
# Second id from stub library should be 0x11111112
self.assertEqual(0x11111112, fileid2)
# Did the files both get stored?
self.failUnless(self.storage.hasFile(fileid1))
self.failUnless(self.storage.hasFile(fileid2))
def test_hashes(self):
# Check that the MD5, SHA1 and SHA256 hashes are correct.
data = 'i am some data'
md5 = hashlib.md5(data).hexdigest()
sha1 = hashlib.sha1(data).hexdigest()
sha256 = hashlib.sha256(data).hexdigest()
newfile = self.storage.startAddFile('file', len(data))
newfile.append(data)
lfc_id, lfa_id = newfile.store()
lfc = LibraryFileContent.get(lfc_id)
self.assertEqual(md5, lfc.md5)
self.assertEqual(sha1, lfc.sha1)
self.assertEqual(sha256, lfc.sha256)
def setUp(self):
switch_dbuser('librarian')
self.directory = tempfile.mkdtemp()
self.storage = LibrarianStorage(self.directory, db.Library())
class LibrarianStorageDBTests(unittest.TestCase):
layer = LaunchpadZopelessLayer
def setUp(self):
switch_dbuser('librarian')
self.directory = tempfile.mkdtemp()
self.storage = LibrarianStorage(self.directory, db.Library())
def tearDown(self):
shutil.rmtree(self.directory, ignore_errors=True)
def test_addFile(self):
data = 'data ' * 50
digest = hashlib.sha1(data).hexdigest()
newfile = self.storage.startAddFile('file1', len(data))
newfile.srcDigest = digest
newfile.append(data)
fileid, aliasid = newfile.store()
self.failUnless(self.storage.hasFile(fileid))
def test_addFiles_identical(self):
# Start adding two files with identical data
data = 'data ' * 5000
newfile1 = self.storage.startAddFile('file1', len(data))
newfile2 = self.storage.startAddFile('file2', len(data))
newfile1.append(data)
newfile2.append(data)
id1, alias1 = newfile1.store()
id2, alias2 = newfile2.store()
# Make sure we actually got an id
self.assertNotEqual(None, id1)
self.assertNotEqual(None, id2)
# But they are two different ids, because we leave duplicate handling
# to the garbage collector
self.failIfEqual(id1, id2)
def test_badDigest(self):
data = 'data ' * 50
digest = 'crud'
newfile = self.storage.startAddFile('file', len(data))
newfile.srcDigest = digest
newfile.append(data)
self.assertRaises(DigestMismatchError, newfile.store)
def test_alias(self):
# Add a file (and so also add an alias)
data = 'data ' * 50
newfile = self.storage.startAddFile('file1', len(data))
newfile.mimetype = 'text/unknown'
newfile.append(data)
fileid, aliasid = newfile.store()
# Check that its alias has the right mimetype
fa = self.storage.getFileAlias(aliasid, None, '/')
self.assertEqual('text/unknown', fa.mimetype)
# Re-add the same file, with the same name and mimetype...
newfile2 = self.storage.startAddFile('file1', len(data))
newfile2.mimetype = 'text/unknown'
newfile2.append(data)
fileid2, aliasid2 = newfile2.store()
# Verify that we didn't get back the same alias ID
self.assertNotEqual(fa.id,
self.storage.getFileAlias(aliasid2, None, '/').id)
def test_clientProvidedDuplicateIDs(self):
# This test checks the new behaviour specified by LibrarianTransactions
# spec: don't create IDs in DB, but do check they don't exist.
# Create a new file
newfile = LibraryFileUpload(self.storage, 'filename', 0)
# Set a content ID on the file (same as would happen with a
# client-generated ID) and store it
newfile.contentID = 666
newfile.store()
newfile = LibraryFileUpload(self.storage, 'filename', 0)
newfile.contentID = 666
self.assertRaises(DuplicateFileIDError, newfile.store)
def test_clientProvidedDuplicateContent(self):
# Check the new behaviour specified by LibrarianTransactions
# spec: allow duplicate content with distinct IDs.
content = 'some content'
# Store a file with id 6661
newfile1 = LibraryFileUpload(self.storage, 'filename', 0)
newfile1.contentID = 6661
newfile1.append(content)
fileid1, aliasid1 = newfile1.store()
# Store second file identical to the first, with id 6662
newfile2 = LibraryFileUpload(self.storage, 'filename', 0)
newfile2.contentID = 6662
newfile2.append(content)
fileid2, aliasid2 = newfile2.store()
# Create rows in the database for these files.
LibraryFileContent(filesize=0,
sha1='foo',
md5='xx',
sha256='xx',
id=6661)
LibraryFileContent(filesize=0,
sha1='foo',
md5='xx',
sha256='xx',
id=6662)
flush_database_updates()
