def test_DeleteUnreferencedAliases(self):
self.ztm.begin()
# Confirm that our sample files are there.
f1 = LibraryFileAlias.get(self.f1_id)
f2 = LibraryFileAlias.get(self.f2_id)
# Grab the content IDs related to these
# unreferenced LibraryFileAliases
c1_id = f1.contentID
c2_id = f2.contentID
del f1, f2
self.ztm.abort()
# Delete unreferenced aliases
librariangc.delete_unreferenced_aliases(self.con)
# This should have committed
self.ztm.begin()
# Confirm that the LibaryFileContents are still there.
LibraryFileContent.get(c1_id)
LibraryFileContent.get(c2_id)
# But the LibraryFileAliases should be gone
self.assertRaises(SQLObjectNotFound, LibraryFileAlias.get, self.f1_id)
self.assertRaises(SQLObjectNotFound, LibraryFileAlias.get, self.f2_id)
def test_clientProvidedDuplicateContent(self):
# Check the new behaviour specified by LibrarianTransactions
# spec: allow duplicate content with distinct IDs.
content = 'some content'
# Store a file with id 6661
newfile1 = LibraryFileUpload(self.storage, 'filename', 0)
newfile1.contentID = 6661
newfile1.append(content)
fileid1, aliasid1 = newfile1.store()
# Store second file identical to the first, with id 6662
newfile2 = LibraryFileUpload(self.storage, 'filename', 0)
newfile2.contentID = 6662
newfile2.append(content)
fileid2, aliasid2 = newfile2.store()
# Create rows in the database for these files.
LibraryFileContent(filesize=0,
sha1='foo',
md5='xx',
sha256='xx',
id=6661)
LibraryFileContent(filesize=0,
sha1='foo',
md5='xx',
sha256='xx',
id=6662)
flush_database_updates()
def _makeLibraryFileContent(self, content):
"""Create a `LibraryFileContent`."""
size = len(content)
md5 = hashlib.md5(content).hexdigest()
sha1 = hashlib.sha1(content).hexdigest()
sha256 = hashlib.sha256(content).hexdigest()
content_object = LibraryFileContent(filesize=size,
md5=md5,
sha1=sha1,
sha256=sha256)
return content_object
def fillLibrarianFile(fileid, content=None):
"""Write contents in disk for a librarian sampledata."""
if content is None:
content = 'x' * LibraryFileContent.get(fileid).filesize
filepath = os.path.join(
config.librarian_server.root, _relFileLocation(fileid))
if not os.path.exists(os.path.dirname(filepath)):
os.makedirs(os.path.dirname(filepath))
with open(filepath, 'wb') as libfile:
libfile.write(content)
def test_hashes(self):
# Check that the MD5, SHA1 and SHA256 hashes are correct.
data = 'i am some data'
md5 = hashlib.md5(data).hexdigest()
sha1 = hashlib.sha1(data).hexdigest()
sha256 = hashlib.sha256(data).hexdigest()
newfile = self.storage.startAddFile('file', len(data))
newfile.append(data)
lfc_id, lfa_id = newfile.store()
lfc = LibraryFileContent.get(lfc_id)
self.assertEqual(md5, lfc.md5)
self.assertEqual(sha1, lfc.sha1)
self.assertEqual(sha256, lfc.sha256)
def test_hashes(self):
# Check that the MD5, SHA1 and SHA256 hashes are correct.
data = 'i am some data'
md5 = hashlib.md5(data).hexdigest()
sha1 = hashlib.sha1(data).hexdigest()
sha256 = hashlib.sha256(data).hexdigest()
newfile = self.storage.startAddFile('file', len(data))
newfile.append(data)
lfc_id, lfa_id = newfile.store()
lfc = LibraryFileContent.get(lfc_id)
self.assertEqual(md5, lfc.md5)
self.assertEqual(sha1, lfc.sha1)
self.assertEqual(sha256, lfc.sha256)
def lookupBySHA1(self, digest):
return [fc.id for fc in LibraryFileContent.selectBy(sha1=digest)]
def add(self, digest, size, md5_digest, sha256_digest):
lfc = LibraryFileContent(filesize=size,
sha1=digest,
md5=md5_digest,
sha256=sha256_digest)
return lfc.id
def addFile(self,
name,
size,
file,
contentType,
expires=None,
debugID=None,
allow_zero_length=False):
"""Add a file to the librarian.
:param name: Name to store the file as
:param size: Size of the file
:param file: File-like object with the content in it
:param contentType: mime-type, e.g. text/plain
:param expires: Expiry time of file. See LibrarianGarbageCollection.
Set to None to only expire when it is no longer referenced.
:param debugID: Optional. If set, causes extra logging for this
request on the server, which will be marked with the value
given.
:param allow_zero_length: If True permit zero length files.
:returns: aliasID as an integer
:raises UploadFailed: If the server rejects the upload for some
reason.
"""
if file is None:
raise TypeError('Bad File Descriptor: %s' % repr(file))
if allow_zero_length:
min_size = -1
else:
min_size = 0
if size <= min_size:
raise UploadFailed('Invalid length: %d' % size)
name = six.ensure_binary(name)
# Import in this method to avoid a circular import
from lp.services.librarian.model import LibraryFileContent
from lp.services.librarian.model import LibraryFileAlias
self._connect()
try:
# Get the name of the database the client is using, so that
# the server can check that the client is using the same
# database as the server.
store = IMasterStore(LibraryFileAlias)
databaseName = self._getDatabaseName(store)
# Generate new content and alias IDs.
# (we'll create rows with these IDs later, but not yet)
contentID = store.execute(
"SELECT nextval('libraryfilecontent_id_seq')").get_one()[0]
aliasID = store.execute(
"SELECT nextval('libraryfilealias_id_seq')").get_one()[0]
# Send command
self._sendLine('STORE %d %s' % (size, name))
# Send headers
self._sendHeader('Database-Name', databaseName)
self._sendHeader('File-Content-ID', contentID)
self._sendHeader('File-Alias-ID', aliasID)
if debugID is not None:
self._sendHeader('Debug-ID', debugID)
# Send blank line. Do not check for a response from the
# server when no data will be sent. Otherwise
# _checkError() might consume the "200" response which
# is supposed to be read below in this method.
self._sendLine('', check_for_error_responses=(size > 0))
# Prepare to the upload the file
md5_digester = hashlib.md5()
sha1_digester = hashlib.sha1()
sha256_digester = hashlib.sha256()
bytesWritten = 0
# Read in and upload the file 64kb at a time, by using the two-arg
# form of iter (see
# /usr/share/doc/python/html/library/functions.html#iter).
for chunk in iter(lambda: file.read(1024 * 64), ''):
self.state.f.write(chunk)
bytesWritten += len(chunk)
md5_digester.update(chunk)
sha1_digester.update(chunk)
sha256_digester.update(chunk)
assert bytesWritten == size, (
'size is %d, but %d were read from the file' %
(size, bytesWritten))
self.state.f.flush()
# Read response
response = self.state.f.readline().strip()
if response != '200':
raise UploadFailed('Server said: ' + response)
# Add rows to DB
content = LibraryFileContent(id=contentID,
filesize=size,
sha256=sha256_digester.hexdigest(),
sha1=sha1_digester.hexdigest(),
md5=md5_digester.hexdigest())
LibraryFileAlias(id=aliasID,
content=content,
filename=name.decode('UTF-8'),
mimetype=contentType,
expires=expires,
restricted=self.restricted)
Store.of(content).flush()
assert isinstance(aliasID, (int, long)), \
"aliasID %r not an integer" % (aliasID, )
return aliasID
finally:
self._close()
def lookupBySHA1(self, digest):
return [fc.id for fc in LibraryFileContent.selectBy(sha1=digest)]