Example #1
0
def decode(segment):
    """ Decode the NZBSegment's articleData to it's destination. Toggle the NZBSegment
    instance as having been decoded, then assemble all the segments together if all their
    decoded segment filenames exist """
    if Hellanzb.SHUTDOWN:
        return

    encoding = UNKNOWN
    try:
        segment.loadArticleData()
        encoding, encodingMessage = decodeArticleData(segment)
        
    except OutOfDiskSpace:
        # Ran out of disk space and the download was paused! Easiest way out of this
        # sticky situation is to requeue the segment
        nuke(segment.getDestination())
        segment.nzbFile.totalReadBytes -= segment.readBytes
        if not segment.cachedToDisk:
            segment.nzbFile.nzb.cachedArticleDataBytes -= segment.readBytes
        segment.nzbFile.nzb.totalReadBytes -= segment.readBytes
        reactor.callFromThread(segment.fromQueue.put, (segment.priority, segment))
        return
    except Exception, e:
        if handleCanceledSegment(segment):
            # Cancelled NZBs could potentially cause IOErrors during writes -- just handle
            # cleanup and return
            return

        error(segment.nzbFile.showFilename + ' segment: ' + str(segment.number) + \
              ' a problem occurred during decoding', e)
        del segment.articleData
        segment.articleData = ''
        if not segment.cachedToDisk:
            segment.nzbFile.nzb.cachedArticleDataBytes -= segment.readBytes
        touch(segment.getDestination())
Example #2
0
def decode(segment):
    """ Decode the NZBSegment's articleData to it's destination. Toggle the NZBSegment
    instance as having been decoded, then assemble all the segments together if all their
    decoded segment filenames exist """
    if Hellanzb.SHUTDOWN:
        return

    encoding = UNKNOWN
    try:
        segment.loadArticleData()
        encoding, encodingMessage = decodeArticleData(segment)
        
    except OutOfDiskSpace:
        # Ran out of disk space and the download was paused! Easiest way out of this
        # sticky situation is to requeue the segment
        nuke(segment.getDestination())
        segment.nzbFile.totalReadBytes -= segment.readBytes
        if not segment.cachedToDisk:
            segment.nzbFile.nzb.cachedArticleDataBytes -= segment.readBytes
        segment.nzbFile.nzb.totalReadBytes -= segment.readBytes
        reactor.callFromThread(segment.fromQueue.put, (segment.priority, segment))
        return
    except Exception, e:
        if handleCanceledSegment(segment):
            # Cancelled NZBs could potentially cause IOErrors during writes -- just handle
            # cleanup and return
            return

        error(segment.nzbFile.showFilename + ' segment: ' + str(segment.number) + \
              ' a problem occurred during decoding', e)
        del segment.articleData
        segment.articleData = ''
        if not segment.cachedToDisk:
            segment.nzbFile.nzb.cachedArticleDataBytes -= segment.readBytes
        touch(segment.getDestination())
Example #3
0
def handleCanceledFile(nzbFile):
    """ Return whether or not the specified NZBFile has been canceled. If so, delete its
    associated decoded file on disk, if it exists """
    if nzbFile.nzb.isCanceled():
        nuke(nzbFile.getDestination())
        return True
    return False
Example #4
0
def handleCanceledFile(nzbFile):
    """ Return whether or not the specified NZBFile has been canceled. If so, delete its
    associated decoded file on disk, if it exists """
    if nzbFile.nzb.isCanceled():
        nuke(nzbFile.getDestination())
        return True
    return False
Example #5
0
def crcFailedRequeue(segment, encodingMessage):
    """ Requeue a segment that failed the CRC verification for download via the twisted main
    thread """
    restartedDownloader = False
    try:
        if not Hellanzb.downloading:
            # We need to requeue after downloading everything in this NZB. Readd the NZB
            # to the queue, for tryFinishNZB
            debug('crcFailedRequeue: Downloader stopped and we need to attempt requeue: '
                  'restarting downloader')
            beginDownload(segment.nzbFile.nzb)
            restartedDownloader = True

        Hellanzb.queue.requeueMissing(segment.fromServer.factory, segment)
    except PoolsExhausted:
        if restartedDownloader and len(Hellanzb.queue.nzbs) == 1 and \
                segment.nzbFile.nzb in Hellanzb.queue.nzbs:
            # We restarted the downloader for no reason -- stop it
            debug('crcFailedRequeue: Restarted downloader and caught PoolsExhausted: '
                  ' stopping downloader')
            endDownload()

        # All servers failed to get a good copy of this segment
        error(encodingMessage)

        # Acquire the assembly lock to avoid potential clashing with postpone() 
        segment.nzbFile.nzb.assembleLock.acquire()

        if len(segment.failedServerPools) > 1:
            # Use the largest file (by size) downloaded from the servers, and delete the
            # rest
            failedFiles = []
            for serverPoolName in segment.failedServerPools:
                failedFile = segment.getDestination() + '-hellafailed_%s' % serverPoolName
                if not os.path.exists(failedFile):
                    # Failed files won't exist in the case the server reported
                    # the article as missing
                    continue
                failedFiles.append((os.path.getsize(failedFile), failedFile))
            failedFiles.sort()
            useFile = failedFiles.pop()[1]
            for failedFile in failedFiles:
                nuke(failedFile[1])
        else:
            # FIXME: crcFailedRequeue should really never be triggered when there's only
            # one server to fail on
            useFile = segment.getDestination() + '-hellafailed_%s' % segment.failedServerPools[0]
        os.rename(useFile, segment.getDestination())

        segment.nzbFile.nzb.assembleLock.release()

        postDecode(segment)
    else:
        debug('%s from server: %s. requeued to alternate server' % \
              (encodingMessage, segment.fromServer.factory.serverPoolName))
Example #6
0
def crcFailedRequeue(segment, encodingMessage):
    """ Requeue a segment that failed the CRC verification for download via the twisted main
    thread """
    restartedDownloader = False
    try:
        if not Hellanzb.downloading:
            # We need to requeue after downloading everything in this NZB. Readd the NZB
            # to the queue, for tryFinishNZB
            debug('crcFailedRequeue: Downloader stopped and we need to attempt requeue: '
                  'restarting downloader')
            beginDownload(segment.nzbFile.nzb)
            restartedDownloader = True

        Hellanzb.queue.requeueMissing(segment.fromServer.factory, segment)
    except PoolsExhausted:
        if restartedDownloader and len(Hellanzb.queue.nzbs) == 1 and \
                segment.nzbFile.nzb in Hellanzb.queue.nzbs:
            # We restarted the downloader for no reason -- stop it
            debug('crcFailedRequeue: Restarted downloader and caught PoolsExhausted: '
                  ' stopping downloader')
            endDownload()

        # All servers failed to get a good copy of this segment
        error(encodingMessage)

        # Acquire the assembly lock to avoid potential clashing with postpone() 
        segment.nzbFile.nzb.assembleLock.acquire()

        if len(segment.failedServerPools) > 1:
            # Use the largest file (by size) downloaded from the servers, and delete the
            # rest
            failedFiles = []
            for serverPoolName in segment.failedServerPools:
                failedFile = segment.getDestination() + '-hellafailed_%s' % serverPoolName
                if not os.path.exists(failedFile):
                    # Failed files won't exist in the case the server reported
                    # the article as missing
                    continue
                failedFiles.append((os.path.getsize(failedFile), failedFile))
            failedFiles.sort()
            useFile = failedFiles.pop()[1]
            for failedFile in failedFiles:
                nuke(failedFile[1])
        else:
            # FIXME: crcFailedRequeue should really never be triggered when there's only
            # one server to fail on
            useFile = segment.getDestination() + '-hellafailed_%s' % segment.failedServerPools[0]
        os.rename(useFile, segment.getDestination())

        segment.nzbFile.nzb.assembleLock.release()

        postDecode(segment)
    else:
        debug('%s from server: %s. requeued to alternate server' % \
              (encodingMessage, segment.fromServer.factory.serverPoolName))
Example #7
0
    def loadArticleDataFromDisk(self):
        """ Load the previously downloaded article BODY from disk, as a list to the .articleData
        variable. Removes the on disk version upon loading """
        # downloaded encodedDataFile was written to disk by NZBLeecher
        encodedDataFile = open(os.path.join(Hellanzb.DOWNLOAD_TEMP_DIR, self.getTempFileName() + "_ENC"), "rb")
        # remove crlfs. FIXME: might be quicker to do this during a later loop
        self.articleData = [line[:-2] for line in encodedDataFile]
        encodedDataFile.close()

        # Delete the copy on disk ASAP
        nuke(os.path.join(Hellanzb.DOWNLOAD_TEMP_DIR, self.getTempFileName() + "_ENC"))
Example #8
0
    def loadArticleDataFromDisk(self):
        """ Load the previously downloaded article BODY from disk, as a list to the .articleData
        variable. Removes the on disk version upon loading """
        # downloaded encodedDataFile was written to disk by NZBLeecher
        encodedDataFile = open(
            os.path.join(Hellanzb.DOWNLOAD_TEMP_DIR,
                         self.getTempFileName() + '_ENC'), 'rb')
        # remove crlfs. FIXME: might be quicker to do this during a later loop
        self.articleData = [line[:-2] for line in encodedDataFile]
        encodedDataFile.close()

        # Delete the copy on disk ASAP
        nuke(
            os.path.join(Hellanzb.DOWNLOAD_TEMP_DIR,
                         self.getTempFileName() + '_ENC'))
Example #9
0
def tryAssemble(nzbFile):
    """ Assemble the specified NZBFile if all its segments have been downloaded """
    if nzbFile.isAllSegmentsDecoded():
        try:
            assembleNZBFile(nzbFile)
            # NOTE: exceptions here might cause Hellanzb.queue.fileDone() to not be
            # called
        except OutOfDiskSpace:
            # Delete the partially assembled file. It will be re-assembled later when the
            # downloader becomes unpaused
            nuke(nzbFile.getDestination())
            nzbFile.interruptedAssembly = True
        except SystemExit, se:
            # checkShutdown() throws this, let the thread die
            pass
        except Exception, e:
            # Cancelled NZBs could potentially cause IOErrors during writes -- just handle
            # cleanup and return
            if not handleCanceledFile(nzbFile):
                raise
Example #10
0
def tryAssemble(nzbFile):
    """ Assemble the specified NZBFile if all its segments have been downloaded """
    if nzbFile.isAllSegmentsDecoded():
        try:
            assembleNZBFile(nzbFile)
            # NOTE: exceptions here might cause Hellanzb.queue.fileDone() to not be
            # called
        except OutOfDiskSpace:
            # Delete the partially assembled file. It will be re-assembled later when the
            # downloader becomes unpaused
            nuke(nzbFile.getDestination())
            nzbFile.interruptedAssembly = True
        except SystemExit, se:
            # checkShutdown() throws this, let the thread die
            pass
        except Exception, e:
            # Cancelled NZBs could potentially cause IOErrors during writes -- just handle
            # cleanup and return
            if not handleCanceledFile(nzbFile):
                raise