def __init__(self, config, btPersister, ident, torrent, pathprefix): self.config = config self.btPersister = btPersister self.torrent = torrent self.pathprefix = pathprefix #loading self.loaded = False self.shouldAbortLoad = False self.loadLock = threading.Lock() #other shouldPersist = self.config.get('storage', 'persistPieceStatus') self.ownStatus = PersistentOwnStatus(btPersister, shouldPersist, self.torrent.getTotalAmountOfPieces()) self.log = Logger('Storage', '%-6s - ', ident) self.lock = threading.Lock()
class Storage: def __init__(self, config, btPersister, ident, torrent, pathprefix): self.config = config self.btPersister = btPersister self.torrent = torrent self.pathprefix = pathprefix #loading self.loaded = False self.shouldAbortLoad = False self.loadLock = threading.Lock() #other shouldPersist = self.config.get('storage', 'persistPieceStatus') self.ownStatus = PersistentOwnStatus(btPersister, shouldPersist, self.torrent.getTotalAmountOfPieces()) self.log = Logger('Storage', '%-6s - ', ident) self.lock = threading.Lock() ##internal functions - files def _getFilePath(self, filePath): realFilePath = os.path.normpath(os.path.join(self.pathprefix, filePath)) if not realFilePath.startswith(self.pathprefix): raise StorageException('Security violation: file "%s" is not inside base directory "%s" (original path: "%s")', realFilePath, self.pathprefix, os.path.join(self.pathprefix, filePath)) return realFilePath ##internal functions - loading def _checkFile(self, filePath, wantedFileSize): #checks path to file and the size of the file itself, #may throw StorageException if the file path is not acceptable or a directory or file operation fails created = False modified = False #get file path realFilePath = self._getFilePath(filePath) #check directory dirPath = os.path.dirname(realFilePath) if not os.path.exists(dirPath): #directory doesn't exist, create it created = True self.log.debug('Creating Directory "%s"', dirPath) try: os.makedirs(dirPath) except OSError: raise StorageException('Failed to create directory "%s":\n%s' % (dirPath, logTraceback())) #check file if not os.path.exists(realFilePath): #file needs to be created fl = open(realFilePath, 'ab') fl.close() created = True self.log.debug('Processing file "%s" (original name "%s"): new "%s", isdir "%s", isfile "%s", islink "%s", dirname "%s", basename "%s"',\ realFilePath, filePath, str(created), str(os.path.isdir(realFilePath)), str(os.path.isfile(realFilePath)),\ str(os.path.islink(realFilePath)), dirPath, os.path.basename(realFilePath)) try: fl = open(realFilePath, 'rb+') with fl: #file opened fl.seek(0, 2) currentFileSize = fl.tell() if currentFileSize < wantedFileSize: self.log.debug('File "%s" is %d bytes to short', realFilePath, wantedFileSize - currentFileSize) modified = True else: self.log.debug('File "%s" has the correct size', realFilePath) #fill if needed if (not self.shouldAbortLoad) and currentFileSize + 1045876 < wantedFileSize: #large fill start = time() fl.seek(1048575, 1) fl.write('\x00') fl.flush() currentFileSize = fl.tell() needed = time() - start try: step = int(1048575 * 0.1 / needed) except ZeroDivisionError: step = wantedFileSize - currentFileSize - 1 step = max(1, step) self.log.debug('Needed %f seconds for 1 Mb, step size %i', needed, step) while (not self.shouldAbortLoad) and currentFileSize + step <= wantedFileSize - 1: fl.seek(step, 1) fl.write('\x00') fl.flush() currentFileSize = fl.tell() self.log.debug("Progress: %i / %i", currentFileSize, wantedFileSize) if (not self.shouldAbortLoad) and currentFileSize < wantedFileSize: #seek remaining bytes and write last byte fl.seek((wantedFileSize - currentFileSize - 1), 1) fl.write('\x00') fl.flush() currentFileSize = fl.tell() self.log.debug("Progress: %i / %i", currentFileSize, wantedFileSize) except IOError: #something failed raise StorageException('Failure while processing file "%s":\n%s' % (realFilePath, logTraceback())) return created, modified def _checkAllFiles(self): anyModified = False allCreated = True files = self.torrent.getFiles() place = 0 while (not self.shouldAbortLoad) and place < len(files): fileSet = files[place] created, modified = self._checkFile(fileSet['path'], fileSet['size']) anyModified |= modified allCreated &= created place += 1 return allCreated, anyModified def _checkPieceAvailability(self): #check which pieces are already finished #may throw StorageException if things go wrong piece = 0 pieceAmount = self.torrent.getTotalAmountOfPieces() while (not self.shouldAbortLoad) and piece < pieceAmount: #check piece data = self.getData(piece, 0, self.torrent.getLengthOfPiece(piece)) if sha1(data).digest() == self.torrent.getPieceHashByPieceIndex(piece): #hash matches, piece is finished self.ownStatus.setPieceStatus(piece, True) self.log.debug('Piece Nr. %d is finished', piece) else: #hash doesn't match, not finished self.ownStatus.setPieceStatus(piece, False) self.log.debug('Piece Nr. %d is not finished', piece) piece += 1 def _load(self, completionCallback): with self.loadLock: #inside lock self.log.info('Loading files of torrent') loadSuccess = False try: #check files of torrent if not self.config.get('storage', 'skipFileCheck'): #skipping is not allowed self.log.debug('Not allowed to skip file checking, starting check') allCreated, anyModified = self._checkAllFiles() self.btPersister.store('Storage-checkedFiles', True) elif not self.btPersister.get('Storage-checkedFiles', False): #skipping would be allowed but we didn't check even once up to now self.log.debug('Files were not checked up to now, starting check') allCreated, anyModified = self._checkAllFiles() self.btPersister.store('Storage-checkedFiles', True) else: #skipping is allowed and files were already checked self.log.debug('Skipping file checking') allCreated = False anyModified = False #check which pieces are already finished if allCreated: #no need to check piece availability, all files were just written to disk self.log.debug('Skipping hashing, files were just created') else: #possibly need to check, some files already existed if self.ownStatus.loadPersistedData(): #persisted status info existed self.log.debug('Skipping hashing, managed to load persisted status data') else: #there is no persisted data self.log.debug('Checking which pieces are already finished') self._checkPieceAvailability() #check if loading wasn't aborted if not self.shouldAbortLoad: self.ownStatus.persist() loadSuccess = True self.loaded = True except StorageException, se: self.log.error('Failure during load:\n%s', logTraceback()) if not self.shouldAbortLoad: completionCallback(loadSuccess)