def __init__(self, config, btPersister, ident, torrent, pathprefix):
self.config = config
self.btPersister = btPersister
self.torrent = torrent
self.pathprefix = pathprefix
#loading
self.loaded = False
self.shouldAbortLoad = False
self.loadLock = threading.Lock()
#other
shouldPersist = self.config.get('storage', 'persistPieceStatus')
self.ownStatus = PersistentOwnStatus(btPersister, shouldPersist, self.torrent.getTotalAmountOfPieces())
self.log = Logger('Storage', '%-6s - ', ident)
self.lock = threading.Lock()
class Storage:
def __init__(self, config, btPersister, ident, torrent, pathprefix):
self.config = config
self.btPersister = btPersister
self.torrent = torrent
self.pathprefix = pathprefix
#loading
self.loaded = False
self.shouldAbortLoad = False
self.loadLock = threading.Lock()
#other
shouldPersist = self.config.get('storage', 'persistPieceStatus')
self.ownStatus = PersistentOwnStatus(btPersister, shouldPersist, self.torrent.getTotalAmountOfPieces())
self.log = Logger('Storage', '%-6s - ', ident)
self.lock = threading.Lock()
##internal functions - files
def _getFilePath(self, filePath):
realFilePath = os.path.normpath(os.path.join(self.pathprefix, filePath))
if not realFilePath.startswith(self.pathprefix):
raise StorageException('Security violation: file "%s" is not inside base directory "%s" (original path: "%s")', realFilePath, self.pathprefix, os.path.join(self.pathprefix, filePath))
return realFilePath
##internal functions - loading
def _checkFile(self, filePath, wantedFileSize):
#checks path to file and the size of the file itself,
#may throw StorageException if the file path is not acceptable or a directory or file operation fails
created = False
modified = False
#get file path
realFilePath = self._getFilePath(filePath)
#check directory
dirPath = os.path.dirname(realFilePath)
if not os.path.exists(dirPath):
#directory doesn't exist, create it
created = True
self.log.debug('Creating Directory "%s"', dirPath)
try:
os.makedirs(dirPath)
except OSError:
raise StorageException('Failed to create directory "%s":\n%s' % (dirPath, logTraceback()))
#check file
if not os.path.exists(realFilePath):
#file needs to be created
fl = open(realFilePath, 'ab')
fl.close()
created = True
self.log.debug('Processing file "%s" (original name "%s"): new "%s", isdir "%s", isfile "%s", islink "%s", dirname "%s", basename "%s"',\
realFilePath, filePath, str(created), str(os.path.isdir(realFilePath)), str(os.path.isfile(realFilePath)),\
str(os.path.islink(realFilePath)), dirPath, os.path.basename(realFilePath))
try:
fl = open(realFilePath, 'rb+')
with fl:
#file opened
fl.seek(0, 2)
currentFileSize = fl.tell()
if currentFileSize < wantedFileSize:
self.log.debug('File "%s" is %d bytes to short', realFilePath, wantedFileSize - currentFileSize)
modified = True
else:
self.log.debug('File "%s" has the correct size', realFilePath)
#fill if needed
if (not self.shouldAbortLoad) and currentFileSize + 1045876 < wantedFileSize:
#large fill
start = time()
fl.seek(1048575, 1)
fl.write('\x00')
fl.flush()
currentFileSize = fl.tell()
needed = time() - start
try:
step = int(1048575 * 0.1 / needed)
except ZeroDivisionError:
step = wantedFileSize - currentFileSize - 1
step = max(1, step)
self.log.debug('Needed %f seconds for 1 Mb, step size %i', needed, step)
while (not self.shouldAbortLoad) and currentFileSize + step <= wantedFileSize - 1:
fl.seek(step, 1)
fl.write('\x00')
fl.flush()
currentFileSize = fl.tell()
self.log.debug("Progress: %i / %i", currentFileSize, wantedFileSize)
if (not self.shouldAbortLoad) and currentFileSize < wantedFileSize:
#seek remaining bytes and write last byte
fl.seek((wantedFileSize - currentFileSize - 1), 1)
fl.write('\x00')
fl.flush()
currentFileSize = fl.tell()
self.log.debug("Progress: %i / %i", currentFileSize, wantedFileSize)
except IOError:
#something failed
raise StorageException('Failure while processing file "%s":\n%s' % (realFilePath, logTraceback()))
return created, modified
def _checkAllFiles(self):
anyModified = False
allCreated = True
files = self.torrent.getFiles()
place = 0
while (not self.shouldAbortLoad) and place < len(files):
fileSet = files[place]
created, modified = self._checkFile(fileSet['path'], fileSet['size'])
anyModified |= modified
allCreated &= created
place += 1
return allCreated, anyModified
def _checkPieceAvailability(self):
#check which pieces are already finished
#may throw StorageException if things go wrong
piece = 0
pieceAmount = self.torrent.getTotalAmountOfPieces()
while (not self.shouldAbortLoad) and piece < pieceAmount:
#check piece
data = self.getData(piece, 0, self.torrent.getLengthOfPiece(piece))
if sha1(data).digest() == self.torrent.getPieceHashByPieceIndex(piece):
#hash matches, piece is finished
self.ownStatus.setPieceStatus(piece, True)
self.log.debug('Piece Nr. %d is finished', piece)
else:
#hash doesn't match, not finished
self.ownStatus.setPieceStatus(piece, False)
self.log.debug('Piece Nr. %d is not finished', piece)
piece += 1
def _load(self, completionCallback):
with self.loadLock:
#inside lock
self.log.info('Loading files of torrent')
loadSuccess = False
try:
#check files of torrent
if not self.config.get('storage', 'skipFileCheck'):
#skipping is not allowed
self.log.debug('Not allowed to skip file checking, starting check')
allCreated, anyModified = self._checkAllFiles()
self.btPersister.store('Storage-checkedFiles', True)
elif not self.btPersister.get('Storage-checkedFiles', False):
#skipping would be allowed but we didn't check even once up to now
self.log.debug('Files were not checked up to now, starting check')
allCreated, anyModified = self._checkAllFiles()
self.btPersister.store('Storage-checkedFiles', True)
else:
#skipping is allowed and files were already checked
self.log.debug('Skipping file checking')
allCreated = False
anyModified = False
#check which pieces are already finished
if allCreated:
#no need to check piece availability, all files were just written to disk
self.log.debug('Skipping hashing, files were just created')
else:
#possibly need to check, some files already existed
if self.ownStatus.loadPersistedData():
#persisted status info existed
self.log.debug('Skipping hashing, managed to load persisted status data')
else:
#there is no persisted data
self.log.debug('Checking which pieces are already finished')
self._checkPieceAvailability()
#check if loading wasn't aborted
if not self.shouldAbortLoad:
self.ownStatus.persist()
loadSuccess = True
self.loaded = True
except StorageException, se:
self.log.error('Failure during load:\n%s', logTraceback())
if not self.shouldAbortLoad:
completionCallback(loadSuccess)
