def test_DeleteUnreferencedAliases(self): self.ztm.begin() # Confirm that our sample files are there. f1 = LibraryFileAlias.get(self.f1_id) f2 = LibraryFileAlias.get(self.f2_id) # Grab the content IDs related to these # unreferenced LibraryFileAliases c1_id = f1.contentID c2_id = f2.contentID del f1, f2 self.ztm.abort() # Delete unreferenced aliases librariangc.delete_unreferenced_aliases(self.con) # This should have committed self.ztm.begin() # Confirm that the LibaryFileContents are still there. LibraryFileContent.get(c1_id) LibraryFileContent.get(c2_id) # But the LibraryFileAliases should be gone self.assertRaises(SQLObjectNotFound, LibraryFileAlias.get, self.f1_id) self.assertRaises(SQLObjectNotFound, LibraryFileAlias.get, self.f2_id)
def test_clientProvidedDuplicateContent(self): # Check the new behaviour specified by LibrarianTransactions # spec: allow duplicate content with distinct IDs. content = 'some content' # Store a file with id 6661 newfile1 = LibraryFileUpload(self.storage, 'filename', 0) newfile1.contentID = 6661 newfile1.append(content) fileid1, aliasid1 = newfile1.store() # Store second file identical to the first, with id 6662 newfile2 = LibraryFileUpload(self.storage, 'filename', 0) newfile2.contentID = 6662 newfile2.append(content) fileid2, aliasid2 = newfile2.store() # Create rows in the database for these files. LibraryFileContent(filesize=0, sha1='foo', md5='xx', sha256='xx', id=6661) LibraryFileContent(filesize=0, sha1='foo', md5='xx', sha256='xx', id=6662) flush_database_updates()
def _makeLibraryFileContent(self, content): """Create a `LibraryFileContent`.""" size = len(content) md5 = hashlib.md5(content).hexdigest() sha1 = hashlib.sha1(content).hexdigest() sha256 = hashlib.sha256(content).hexdigest() content_object = LibraryFileContent(filesize=size, md5=md5, sha1=sha1, sha256=sha256) return content_object
def fillLibrarianFile(fileid, content=None): """Write contents in disk for a librarian sampledata.""" if content is None: content = 'x' * LibraryFileContent.get(fileid).filesize filepath = os.path.join( config.librarian_server.root, _relFileLocation(fileid)) if not os.path.exists(os.path.dirname(filepath)): os.makedirs(os.path.dirname(filepath)) with open(filepath, 'wb') as libfile: libfile.write(content)
def test_hashes(self): # Check that the MD5, SHA1 and SHA256 hashes are correct. data = 'i am some data' md5 = hashlib.md5(data).hexdigest() sha1 = hashlib.sha1(data).hexdigest() sha256 = hashlib.sha256(data).hexdigest() newfile = self.storage.startAddFile('file', len(data)) newfile.append(data) lfc_id, lfa_id = newfile.store() lfc = LibraryFileContent.get(lfc_id) self.assertEqual(md5, lfc.md5) self.assertEqual(sha1, lfc.sha1) self.assertEqual(sha256, lfc.sha256)
def lookupBySHA1(self, digest): return [fc.id for fc in LibraryFileContent.selectBy(sha1=digest)]
def add(self, digest, size, md5_digest, sha256_digest): lfc = LibraryFileContent(filesize=size, sha1=digest, md5=md5_digest, sha256=sha256_digest) return lfc.id
def addFile(self, name, size, file, contentType, expires=None, debugID=None, allow_zero_length=False): """Add a file to the librarian. :param name: Name to store the file as :param size: Size of the file :param file: File-like object with the content in it :param contentType: mime-type, e.g. text/plain :param expires: Expiry time of file. See LibrarianGarbageCollection. Set to None to only expire when it is no longer referenced. :param debugID: Optional. If set, causes extra logging for this request on the server, which will be marked with the value given. :param allow_zero_length: If True permit zero length files. :returns: aliasID as an integer :raises UploadFailed: If the server rejects the upload for some reason. """ if file is None: raise TypeError('Bad File Descriptor: %s' % repr(file)) if allow_zero_length: min_size = -1 else: min_size = 0 if size <= min_size: raise UploadFailed('Invalid length: %d' % size) name = six.ensure_binary(name) # Import in this method to avoid a circular import from lp.services.librarian.model import LibraryFileContent from lp.services.librarian.model import LibraryFileAlias self._connect() try: # Get the name of the database the client is using, so that # the server can check that the client is using the same # database as the server. store = IMasterStore(LibraryFileAlias) databaseName = self._getDatabaseName(store) # Generate new content and alias IDs. # (we'll create rows with these IDs later, but not yet) contentID = store.execute( "SELECT nextval('libraryfilecontent_id_seq')").get_one()[0] aliasID = store.execute( "SELECT nextval('libraryfilealias_id_seq')").get_one()[0] # Send command self._sendLine('STORE %d %s' % (size, name)) # Send headers self._sendHeader('Database-Name', databaseName) self._sendHeader('File-Content-ID', contentID) self._sendHeader('File-Alias-ID', aliasID) if debugID is not None: self._sendHeader('Debug-ID', debugID) # Send blank line. Do not check for a response from the # server when no data will be sent. Otherwise # _checkError() might consume the "200" response which # is supposed to be read below in this method. self._sendLine('', check_for_error_responses=(size > 0)) # Prepare to the upload the file md5_digester = hashlib.md5() sha1_digester = hashlib.sha1() sha256_digester = hashlib.sha256() bytesWritten = 0 # Read in and upload the file 64kb at a time, by using the two-arg # form of iter (see # /usr/share/doc/python/html/library/functions.html#iter). for chunk in iter(lambda: file.read(1024 * 64), ''): self.state.f.write(chunk) bytesWritten += len(chunk) md5_digester.update(chunk) sha1_digester.update(chunk) sha256_digester.update(chunk) assert bytesWritten == size, ( 'size is %d, but %d were read from the file' % (size, bytesWritten)) self.state.f.flush() # Read response response = self.state.f.readline().strip() if response != '200': raise UploadFailed('Server said: ' + response) # Add rows to DB content = LibraryFileContent(id=contentID, filesize=size, sha256=sha256_digester.hexdigest(), sha1=sha1_digester.hexdigest(), md5=md5_digester.hexdigest()) LibraryFileAlias(id=aliasID, content=content, filename=name.decode('UTF-8'), mimetype=contentType, expires=expires, restricted=self.restricted) Store.of(content).flush() assert isinstance(aliasID, (int, long)), \ "aliasID %r not an integer" % (aliasID, ) return aliasID finally: self._close()