def decode(segment):
""" Decode the NZBSegment's articleData to it's destination. Toggle the NZBSegment
instance as having been decoded, then assemble all the segments together if all their
decoded segment filenames exist """
if Hellanzb.SHUTDOWN:
return
encoding = UNKNOWN
try:
segment.loadArticleData()
encoding, encodingMessage = decodeArticleData(segment)
except OutOfDiskSpace:
# Ran out of disk space and the download was paused! Easiest way out of this
# sticky situation is to requeue the segment
nuke(segment.getDestination())
segment.nzbFile.totalReadBytes -= segment.readBytes
if not segment.cachedToDisk:
segment.nzbFile.nzb.cachedArticleDataBytes -= segment.readBytes
segment.nzbFile.nzb.totalReadBytes -= segment.readBytes
reactor.callFromThread(segment.fromQueue.put, (segment.priority, segment))
return
except Exception, e:
if handleCanceledSegment(segment):
# Cancelled NZBs could potentially cause IOErrors during writes -- just handle
# cleanup and return
return
error(segment.nzbFile.showFilename + ' segment: ' + str(segment.number) + \
' a problem occurred during decoding', e)
del segment.articleData
segment.articleData = ''
if not segment.cachedToDisk:
segment.nzbFile.nzb.cachedArticleDataBytes -= segment.readBytes
touch(segment.getDestination())
def decode(segment):
""" Decode the NZBSegment's articleData to it's destination. Toggle the NZBSegment
instance as having been decoded, then assemble all the segments together if all their
decoded segment filenames exist """
if Hellanzb.SHUTDOWN:
return
encoding = UNKNOWN
try:
segment.loadArticleData()
encoding, encodingMessage = decodeArticleData(segment)
except OutOfDiskSpace:
# Ran out of disk space and the download was paused! Easiest way out of this
# sticky situation is to requeue the segment
nuke(segment.getDestination())
segment.nzbFile.totalReadBytes -= segment.readBytes
if not segment.cachedToDisk:
segment.nzbFile.nzb.cachedArticleDataBytes -= segment.readBytes
segment.nzbFile.nzb.totalReadBytes -= segment.readBytes
reactor.callFromThread(segment.fromQueue.put, (segment.priority, segment))
return
except Exception, e:
if handleCanceledSegment(segment):
# Cancelled NZBs could potentially cause IOErrors during writes -- just handle
# cleanup and return
return
error(segment.nzbFile.showFilename + ' segment: ' + str(segment.number) + \
' a problem occurred during decoding', e)
del segment.articleData
segment.articleData = ''
if not segment.cachedToDisk:
segment.nzbFile.nzb.cachedArticleDataBytes -= segment.readBytes
touch(segment.getDestination())
def handleCanceledFile(nzbFile):
""" Return whether or not the specified NZBFile has been canceled. If so, delete its
associated decoded file on disk, if it exists """
if nzbFile.nzb.isCanceled():
nuke(nzbFile.getDestination())
return True
return False
def handleCanceledFile(nzbFile):
""" Return whether or not the specified NZBFile has been canceled. If so, delete its
associated decoded file on disk, if it exists """
if nzbFile.nzb.isCanceled():
nuke(nzbFile.getDestination())
return True
return False
def crcFailedRequeue(segment, encodingMessage):
""" Requeue a segment that failed the CRC verification for download via the twisted main
thread """
restartedDownloader = False
try:
if not Hellanzb.downloading:
# We need to requeue after downloading everything in this NZB. Readd the NZB
# to the queue, for tryFinishNZB
debug('crcFailedRequeue: Downloader stopped and we need to attempt requeue: '
'restarting downloader')
beginDownload(segment.nzbFile.nzb)
restartedDownloader = True
Hellanzb.queue.requeueMissing(segment.fromServer.factory, segment)
except PoolsExhausted:
if restartedDownloader and len(Hellanzb.queue.nzbs) == 1 and \
segment.nzbFile.nzb in Hellanzb.queue.nzbs:
# We restarted the downloader for no reason -- stop it
debug('crcFailedRequeue: Restarted downloader and caught PoolsExhausted: '
' stopping downloader')
endDownload()
# All servers failed to get a good copy of this segment
error(encodingMessage)
# Acquire the assembly lock to avoid potential clashing with postpone()
segment.nzbFile.nzb.assembleLock.acquire()
if len(segment.failedServerPools) > 1:
# Use the largest file (by size) downloaded from the servers, and delete the
# rest
failedFiles = []
for serverPoolName in segment.failedServerPools:
failedFile = segment.getDestination() + '-hellafailed_%s' % serverPoolName
if not os.path.exists(failedFile):
# Failed files won't exist in the case the server reported
# the article as missing
continue
failedFiles.append((os.path.getsize(failedFile), failedFile))
failedFiles.sort()
useFile = failedFiles.pop()[1]
for failedFile in failedFiles:
nuke(failedFile[1])
else:
# FIXME: crcFailedRequeue should really never be triggered when there's only
# one server to fail on
useFile = segment.getDestination() + '-hellafailed_%s' % segment.failedServerPools[0]
os.rename(useFile, segment.getDestination())
segment.nzbFile.nzb.assembleLock.release()
postDecode(segment)
else:
debug('%s from server: %s. requeued to alternate server' % \
(encodingMessage, segment.fromServer.factory.serverPoolName))
def crcFailedRequeue(segment, encodingMessage):
""" Requeue a segment that failed the CRC verification for download via the twisted main
thread """
restartedDownloader = False
try:
if not Hellanzb.downloading:
# We need to requeue after downloading everything in this NZB. Readd the NZB
# to the queue, for tryFinishNZB
debug('crcFailedRequeue: Downloader stopped and we need to attempt requeue: '
'restarting downloader')
beginDownload(segment.nzbFile.nzb)
restartedDownloader = True
Hellanzb.queue.requeueMissing(segment.fromServer.factory, segment)
except PoolsExhausted:
if restartedDownloader and len(Hellanzb.queue.nzbs) == 1 and \
segment.nzbFile.nzb in Hellanzb.queue.nzbs:
# We restarted the downloader for no reason -- stop it
debug('crcFailedRequeue: Restarted downloader and caught PoolsExhausted: '
' stopping downloader')
endDownload()
# All servers failed to get a good copy of this segment
error(encodingMessage)
# Acquire the assembly lock to avoid potential clashing with postpone()
segment.nzbFile.nzb.assembleLock.acquire()
if len(segment.failedServerPools) > 1:
# Use the largest file (by size) downloaded from the servers, and delete the
# rest
failedFiles = []
for serverPoolName in segment.failedServerPools:
failedFile = segment.getDestination() + '-hellafailed_%s' % serverPoolName
if not os.path.exists(failedFile):
# Failed files won't exist in the case the server reported
# the article as missing
continue
failedFiles.append((os.path.getsize(failedFile), failedFile))
failedFiles.sort()
useFile = failedFiles.pop()[1]
for failedFile in failedFiles:
nuke(failedFile[1])
else:
# FIXME: crcFailedRequeue should really never be triggered when there's only
# one server to fail on
useFile = segment.getDestination() + '-hellafailed_%s' % segment.failedServerPools[0]
os.rename(useFile, segment.getDestination())
segment.nzbFile.nzb.assembleLock.release()
postDecode(segment)
else:
debug('%s from server: %s. requeued to alternate server' % \
(encodingMessage, segment.fromServer.factory.serverPoolName))
def loadArticleDataFromDisk(self):
""" Load the previously downloaded article BODY from disk, as a list to the .articleData
variable. Removes the on disk version upon loading """
# downloaded encodedDataFile was written to disk by NZBLeecher
encodedDataFile = open(os.path.join(Hellanzb.DOWNLOAD_TEMP_DIR, self.getTempFileName() + "_ENC"), "rb")
# remove crlfs. FIXME: might be quicker to do this during a later loop
self.articleData = [line[:-2] for line in encodedDataFile]
encodedDataFile.close()
# Delete the copy on disk ASAP
nuke(os.path.join(Hellanzb.DOWNLOAD_TEMP_DIR, self.getTempFileName() + "_ENC"))
def loadArticleDataFromDisk(self):
""" Load the previously downloaded article BODY from disk, as a list to the .articleData
variable. Removes the on disk version upon loading """
# downloaded encodedDataFile was written to disk by NZBLeecher
encodedDataFile = open(
os.path.join(Hellanzb.DOWNLOAD_TEMP_DIR,
self.getTempFileName() + '_ENC'), 'rb')
# remove crlfs. FIXME: might be quicker to do this during a later loop
self.articleData = [line[:-2] for line in encodedDataFile]
encodedDataFile.close()
# Delete the copy on disk ASAP
nuke(
os.path.join(Hellanzb.DOWNLOAD_TEMP_DIR,
self.getTempFileName() + '_ENC'))
def tryAssemble(nzbFile):
""" Assemble the specified NZBFile if all its segments have been downloaded """
if nzbFile.isAllSegmentsDecoded():
try:
assembleNZBFile(nzbFile)
# NOTE: exceptions here might cause Hellanzb.queue.fileDone() to not be
# called
except OutOfDiskSpace:
# Delete the partially assembled file. It will be re-assembled later when the
# downloader becomes unpaused
nuke(nzbFile.getDestination())
nzbFile.interruptedAssembly = True
except SystemExit, se:
# checkShutdown() throws this, let the thread die
pass
except Exception, e:
# Cancelled NZBs could potentially cause IOErrors during writes -- just handle
# cleanup and return
if not handleCanceledFile(nzbFile):
raise
def tryAssemble(nzbFile):
""" Assemble the specified NZBFile if all its segments have been downloaded """
if nzbFile.isAllSegmentsDecoded():
try:
assembleNZBFile(nzbFile)
# NOTE: exceptions here might cause Hellanzb.queue.fileDone() to not be
# called
except OutOfDiskSpace:
# Delete the partially assembled file. It will be re-assembled later when the
# downloader becomes unpaused
nuke(nzbFile.getDestination())
nzbFile.interruptedAssembly = True
except SystemExit, se:
# checkShutdown() throws this, let the thread die
pass
except Exception, e:
# Cancelled NZBs could potentially cause IOErrors during writes -- just handle
# cleanup and return
if not handleCanceledFile(nzbFile):
raise