def resumePostProcessors(): """ Pickup left off Post Processors that were cancelled via CTRL-C """ # FIXME: with the new queue, could kill the processing dir sym links (for windows) from Hellanzb.NZBLeecher.NZBModel import NZB for archiveDirName in os.listdir(Hellanzb.PROCESSING_DIR): if archiveDirName[0] == '.': continue archive = NZB.fromStateXML('processing', archiveDirName) troll = PostProcessor.PostProcessor(archive) info('Resuming post processor: ' + archiveName(archiveDirName)) troll.start()
def assertRecoveredArchiveName(self, name): n = NZB(name) n.rarPassword = rarPassword = '******' archiveName = n.archiveName Hellanzb.queue.nzbAdd(n) self.writeState() del n self.recoverState() n2 = NZB.fromStateXML('downloading', name) self.assertEquals(rarPassword, n2.rarPassword) self.assertEquals(name, n2.nzbFileName) self.assertEquals(archiveName, n2.archiveName)
def assertRecoveredSkippedPar(self, subject): n = NZB('test.nzb') n.isParRecovery = True n.parPrefix = 'test' n.parType = PAR2 Hellanzb.queue.nzbAdd(n) file = NZBFile(subject, 'today', '*****@*****.**', n) file.isSkippedPar = True self.writeState() del n self.recoverState() n2 = NZB.fromStateXML('downloading', 'test') #print str(Hellanzb.recoveredState) #print str(n2.skippedParSubjects) self.assertEquals(True, n2.isSkippedParSubject(subject))
def enqueueNZBs(nzbFileOrFiles, next=False, writeQueue=True, category=None): """ add one or a list of nzb files to the end of the queue """ if isinstance(nzbFileOrFiles, list) or isinstance(nzbFileOrFiles, tuple): newNzbFiles = nzbFileOrFiles else: newNzbFiles = [nzbFileOrFiles] if len(newNzbFiles) == 0: return False for nzbFile in newNzbFiles: if validNZB(nzbFile): if os.path.normpath(os.path.dirname(nzbFile)) != os.path.normpath(Hellanzb.QUEUE_DIR): copy(nzbFile, os.path.join(Hellanzb.QUEUE_DIR, os.path.basename(nzbFile))) nzbFile = os.path.join(Hellanzb.QUEUE_DIR, os.path.basename(nzbFile)) found = False for n in Hellanzb.nzbQueue: if os.path.normpath(n.nzbFileName) == os.path.normpath(nzbFile): found = True error("Unable to add nzb file to queue: " + os.path.basename(nzbFile) + " it already exists!") if found: continue from Hellanzb.NZBLeecher.NZBModel import NZB name = os.path.basename(nzbFile) nzb = NZB.fromStateXML("queued", nzbFile) logMsg = msg = "Found new nzb" extraLog = [] if not next: Hellanzb.nzbQueue.append(nzb) else: Hellanzb.nzbQueue.insert(0, nzb) if nzb.msgid is not None: extraLog.append("msgid: %s" % nzb.msgid) if category: nzb.category = category extraLog.append("category: %s" % category) if extraLog: logMsg += " (%s)" % ", ".join(extraLog) logMsg += ": " msg += ": " info(logMsg + nzb.archiveName) notify("Queue", "hellanzb " + msg, nzb.archiveName, False) # Determine the total bytes of the NZB if it's not already # known. If there's no NZBs in the queue (we're about to parse the # NZB and begin downloading), there's no point in parsing # now. Unless it's the first run of scanQueue (not writeQueue), # then we FIXME: probably need to scan it if nzb.totalBytes == 0 and (len(Hellanzb.queue.currentNZBs()) or not writeQueue): from Hellanzb.NZBLeecher.NZBParser import NZBTotalBytesParser reactor.callInThread(NZBTotalBytesParser.getBytes, nzb) else: try: shutil.move(nzbFile, Hellanzb.TEMP_DIR + os.sep) except (IOError, OSError), e: error("Unable to move invalid NZB: %s out of the way" % nzbFile) debug("Unable to move invalid NZB: %s out of the way" % nzbFile, e)
def scanQueueDir(firstRun=False, justScan=False): """ Find new/resume old NZB download sessions """ # t = time.time() from Hellanzb.NZBLeecher.NZBModel import NZB currentNZBs = [] for file in os.listdir(Hellanzb.CURRENT_DIR): if Hellanzb.NZB_FILE_RE.search(file): currentNZBs.append(os.path.join(Hellanzb.CURRENT_DIR, file)) # See if we're resuming a nzb fetch resuming = False displayNotification = False newNZBs = [] queuedMap = {} for nzb in Hellanzb.nzbQueue: queuedMap[os.path.normpath(nzb.nzbFileName)] = nzb for file in os.listdir(Hellanzb.QUEUE_DIR): if file in Hellanzb.queueDirIgnore: continue if Hellanzb.NZB_FILE_RE.search(file): if os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file)) not in queuedMap: # Delay enqueueing recently modified NZBs if not isOldEnough(os.path.join(Hellanzb.QUEUE_DIR, file)): continue newNZBs.append(os.path.join(Hellanzb.QUEUE_DIR, file)) elif os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file)) in queuedMap: queuedMap.pop(os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file))) elif isOldEnough(os.path.join(Hellanzb.QUEUE_DIR, file)): if not nzbZipSearch(file) and not nzbGzipSearch(file): Hellanzb.queueDirIgnore.append(file) # Remove anything no longer in the queue directory for nzb in queuedMap.itervalues(): Hellanzb.nzbQueue.remove(nzb) if firstRun: # enqueueNZBs() will delete the recovered state. Save it beforehand for sorting queuedRecoveredState = Hellanzb.recoveredState.queued.copy() enqueueNZBs(newNZBs, writeQueue=not firstRun) if firstRun: sortQueueFromRecoveredState(queuedRecoveredState) # e = time.time() - t if justScan: # Done scanning -- don't bother loading a new NZB # debug('Ziplick scanQueueDir (justScan): ' + Hellanzb.QUEUE_DIR + ' TOOK: ' + str(e)) # debug('Ziplick scanQueueDir (justScan): ' + Hellanzb.QUEUE_DIR) Hellanzb.downloadScannerID = reactor.callLater(7, scanQueueDir, False, True) return # else: # debug('Ziplick scanQueueDir: ' + Hellanzb.QUEUE_DIR) if not currentNZBs: if not Hellanzb.nzbQueue: if firstRun: writeStateXML() # Nothing to do, lets wait 5 seconds and start over Hellanzb.downloadScannerID = reactor.callLater(5, scanQueueDir) if not firstRun and not justScan and not Hellanzb.loggedIdleMessage: notify("Queue", "hellanzb", "No more nzbs left to download", False) Hellanzb.loggedIdleMessage = True return # Start the next download nzb = Hellanzb.nzbQueue[0] nzbfilename = os.path.basename(nzb.nzbFileName) del Hellanzb.nzbQueue[0] # nzbfile will always be a absolute filename nzbfile = os.path.join(Hellanzb.QUEUE_DIR, nzbfilename) move(nzbfile, Hellanzb.CURRENT_DIR) if not (len(newNZBs) == 1 and len(Hellanzb.nzbQueue) == 0): # Show what's going to be downloaded next, unless the queue was empty, and we # only found one nzb (The 'Found new nzb' message is enough in that case) displayNotification = True else: # Resume the NZB in the CURRENT_DIR nzbfilename = currentNZBs[0] nzb = NZB.fromStateXML("downloading", nzbfilename) nzbfilename = os.path.basename(nzb.nzbFileName) displayNotification = True del currentNZBs[0] resuming = True nzbfile = os.path.join(Hellanzb.CURRENT_DIR, nzbfilename) nzb.nzbFileName = nzbfile if firstRun: writeStateXML() if resuming: if nzb.isParRecovery: msg = "Resuming par recovery download" else: msg = "Resuming" parseNZB(nzb, msg) elif displayNotification: parseNZB(nzb) else: parseNZB(nzb, quiet=True)
def enqueueNZBs(nzbFileOrFiles, next=False, writeQueue=True, category=None): """ add one or a list of nzb files to the end of the queue """ if isinstance(nzbFileOrFiles, list) or isinstance(nzbFileOrFiles, tuple): newNzbFiles = nzbFileOrFiles else: newNzbFiles = [nzbFileOrFiles] if len(newNzbFiles) == 0: return False for nzbFile in newNzbFiles: if validNZB(nzbFile): if os.path.normpath(os.path.dirname(nzbFile)) != os.path.normpath( Hellanzb.QUEUE_DIR): copy( nzbFile, os.path.join(Hellanzb.QUEUE_DIR, os.path.basename(nzbFile))) nzbFile = os.path.join(Hellanzb.QUEUE_DIR, os.path.basename(nzbFile)) found = False for n in Hellanzb.nzbQueue: if os.path.normpath( n.nzbFileName) == os.path.normpath(nzbFile): found = True error('Unable to add nzb file to queue: ' + os.path.basename(nzbFile) + \ ' it already exists!') if found: continue from Hellanzb.NZBLeecher.NZBModel import NZB name = os.path.basename(nzbFile) nzb = NZB.fromStateXML('queued', nzbFile) logMsg = msg = 'Found new nzb' extraLog = [] if not next: Hellanzb.nzbQueue.append(nzb) else: Hellanzb.nzbQueue.insert(0, nzb) if nzb.msgid is not None: extraLog.append('msgid: %s' % nzb.msgid) if category: nzb.category = category extraLog.append('category: %s' % category) if extraLog: logMsg += ' (%s)' % ', '.join(extraLog) logMsg += ': ' msg += ': ' info(logMsg + nzb.archiveName) notify('Queue', 'hellanzb ' + msg, nzb.archiveName, False) # Determine the total bytes of the NZB if it's not already # known. If there's no NZBs in the queue (we're about to parse the # NZB and begin downloading), there's no point in parsing # now. Unless it's the first run of scanQueue (not writeQueue), # then we FIXME: probably need to scan it if nzb.totalBytes == 0 and (len(Hellanzb.queue.currentNZBs()) or \ not writeQueue): from Hellanzb.NZBLeecher.NZBParser import NZBTotalBytesParser reactor.callInThread(NZBTotalBytesParser.getBytes, nzb) else: try: shutil.move(nzbFile, Hellanzb.TEMP_DIR + os.sep) except (IOError, OSError), e: error('Unable to move invalid NZB: %s out of the way' % nzbFile) debug( 'Unable to move invalid NZB: %s out of the way' % nzbFile, e)
def scanQueueDir(firstRun=False, justScan=False): """ Find new/resume old NZB download sessions """ #t = time.time() from Hellanzb.NZBLeecher.NZBModel import NZB currentNZBs = [] for file in os.listdir(Hellanzb.CURRENT_DIR): if Hellanzb.NZB_FILE_RE.search(file): currentNZBs.append(os.path.join(Hellanzb.CURRENT_DIR, file)) # See if we're resuming a nzb fetch resuming = False displayNotification = False newNZBs = [] queuedMap = {} for nzb in Hellanzb.nzbQueue: queuedMap[os.path.normpath(nzb.nzbFileName)] = nzb for file in os.listdir(Hellanzb.QUEUE_DIR): if file in Hellanzb.queueDirIgnore: continue if Hellanzb.NZB_FILE_RE.search(file): if os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file)) not in queuedMap: # Delay enqueueing recently modified NZBs if not isOldEnough(os.path.join(Hellanzb.QUEUE_DIR, file)): continue newNZBs.append(os.path.join(Hellanzb.QUEUE_DIR, file)) elif os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file)) in queuedMap: queuedMap.pop( os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file))) elif isOldEnough(os.path.join(Hellanzb.QUEUE_DIR, file)): if not nzbZipSearch(file) and not nzbGzipSearch(file): Hellanzb.queueDirIgnore.append(file) # Remove anything no longer in the queue directory for nzb in queuedMap.itervalues(): Hellanzb.nzbQueue.remove(nzb) if firstRun: # enqueueNZBs() will delete the recovered state. Save it beforehand for sorting queuedRecoveredState = Hellanzb.recoveredState.queued.copy() enqueueNZBs(newNZBs, writeQueue=not firstRun) if firstRun: sortQueueFromRecoveredState(queuedRecoveredState) #e = time.time() - t if justScan: # Done scanning -- don't bother loading a new NZB #debug('Ziplick scanQueueDir (justScan): ' + Hellanzb.QUEUE_DIR + ' TOOK: ' + str(e)) #debug('Ziplick scanQueueDir (justScan): ' + Hellanzb.QUEUE_DIR) Hellanzb.downloadScannerID = reactor.callLater(7, scanQueueDir, False, True) return #else: # debug('Ziplick scanQueueDir: ' + Hellanzb.QUEUE_DIR) if not currentNZBs: if not Hellanzb.nzbQueue: if firstRun: writeStateXML() # Nothing to do, lets wait 5 seconds and start over Hellanzb.downloadScannerID = reactor.callLater(5, scanQueueDir) if not firstRun and not justScan and not Hellanzb.loggedIdleMessage: notify('Queue', 'hellanzb', 'No more nzbs left to download', False) Hellanzb.loggedIdleMessage = True return # Start the next download nzb = Hellanzb.nzbQueue[0] nzbfilename = os.path.basename(nzb.nzbFileName) del Hellanzb.nzbQueue[0] # nzbfile will always be a absolute filename nzbfile = os.path.join(Hellanzb.QUEUE_DIR, nzbfilename) move(nzbfile, Hellanzb.CURRENT_DIR) if not (len(newNZBs) == 1 and len(Hellanzb.nzbQueue) == 0): # Show what's going to be downloaded next, unless the queue was empty, and we # only found one nzb (The 'Found new nzb' message is enough in that case) displayNotification = True else: # Resume the NZB in the CURRENT_DIR nzbfilename = currentNZBs[0] nzb = NZB.fromStateXML('downloading', nzbfilename) nzbfilename = os.path.basename(nzb.nzbFileName) displayNotification = True del currentNZBs[0] resuming = True nzbfile = os.path.join(Hellanzb.CURRENT_DIR, nzbfilename) nzb.nzbFileName = nzbfile if firstRun: writeStateXML() if resuming: if nzb.isParRecovery: msg = 'Resuming par recovery download' else: msg = 'Resuming' parseNZB(nzb, msg) elif displayNotification: parseNZB(nzb) else: parseNZB(nzb, quiet=True)