def resumePostProcessors():
""" Pickup left off Post Processors that were cancelled via CTRL-C """
# FIXME: with the new queue, could kill the processing dir sym links (for windows)
from Hellanzb.NZBLeecher.NZBModel import NZB
for archiveDirName in os.listdir(Hellanzb.PROCESSING_DIR):
if archiveDirName[0] == '.':
continue
archive = NZB.fromStateXML('processing', archiveDirName)
troll = PostProcessor.PostProcessor(archive)
info('Resuming post processor: ' + archiveName(archiveDirName))
troll.start()
def assertRecoveredArchiveName(self, name):
n = NZB(name)
n.rarPassword = rarPassword = '******'
archiveName = n.archiveName
Hellanzb.queue.nzbAdd(n)
self.writeState()
del n
self.recoverState()
n2 = NZB.fromStateXML('downloading', name)
self.assertEquals(rarPassword, n2.rarPassword)
self.assertEquals(name, n2.nzbFileName)
self.assertEquals(archiveName, n2.archiveName)
def assertRecoveredArchiveName(self, name):
n = NZB(name)
n.rarPassword = rarPassword = '******'
archiveName = n.archiveName
Hellanzb.queue.nzbAdd(n)
self.writeState()
del n
self.recoverState()
n2 = NZB.fromStateXML('downloading', name)
self.assertEquals(rarPassword, n2.rarPassword)
self.assertEquals(name, n2.nzbFileName)
self.assertEquals(archiveName, n2.archiveName)
def assertRecoveredSkippedPar(self, subject):
n = NZB('test.nzb')
n.isParRecovery = True
n.parPrefix = 'test'
n.parType = PAR2
Hellanzb.queue.nzbAdd(n)
file = NZBFile(subject, 'today', '*****@*****.**', n)
file.isSkippedPar = True
self.writeState()
del n
self.recoverState()
n2 = NZB.fromStateXML('downloading', 'test')
#print str(Hellanzb.recoveredState)
#print str(n2.skippedParSubjects)
self.assertEquals(True, n2.isSkippedParSubject(subject))
def assertRecoveredSkippedPar(self, subject):
n = NZB('test.nzb')
n.isParRecovery = True
n.parPrefix = 'test'
n.parType = PAR2
Hellanzb.queue.nzbAdd(n)
file = NZBFile(subject, 'today', '*****@*****.**', n)
file.isSkippedPar = True
self.writeState()
del n
self.recoverState()
n2 = NZB.fromStateXML('downloading', 'test')
#print str(Hellanzb.recoveredState)
#print str(n2.skippedParSubjects)
self.assertEquals(True, n2.isSkippedParSubject(subject))
def enqueueNZBs(nzbFileOrFiles, next=False, writeQueue=True, category=None):
""" add one or a list of nzb files to the end of the queue """
if isinstance(nzbFileOrFiles, list) or isinstance(nzbFileOrFiles, tuple):
newNzbFiles = nzbFileOrFiles
else:
newNzbFiles = [nzbFileOrFiles]
if len(newNzbFiles) == 0:
return False
for nzbFile in newNzbFiles:
if validNZB(nzbFile):
if os.path.normpath(os.path.dirname(nzbFile)) != os.path.normpath(Hellanzb.QUEUE_DIR):
copy(nzbFile, os.path.join(Hellanzb.QUEUE_DIR, os.path.basename(nzbFile)))
nzbFile = os.path.join(Hellanzb.QUEUE_DIR, os.path.basename(nzbFile))
found = False
for n in Hellanzb.nzbQueue:
if os.path.normpath(n.nzbFileName) == os.path.normpath(nzbFile):
found = True
error("Unable to add nzb file to queue: " + os.path.basename(nzbFile) + " it already exists!")
if found:
continue
from Hellanzb.NZBLeecher.NZBModel import NZB
name = os.path.basename(nzbFile)
nzb = NZB.fromStateXML("queued", nzbFile)
logMsg = msg = "Found new nzb"
extraLog = []
if not next:
Hellanzb.nzbQueue.append(nzb)
else:
Hellanzb.nzbQueue.insert(0, nzb)
if nzb.msgid is not None:
extraLog.append("msgid: %s" % nzb.msgid)
if category:
nzb.category = category
extraLog.append("category: %s" % category)
if extraLog:
logMsg += " (%s)" % ", ".join(extraLog)
logMsg += ": "
msg += ": "
info(logMsg + nzb.archiveName)
notify("Queue", "hellanzb " + msg, nzb.archiveName, False)
# Determine the total bytes of the NZB if it's not already
# known. If there's no NZBs in the queue (we're about to parse the
# NZB and begin downloading), there's no point in parsing
# now. Unless it's the first run of scanQueue (not writeQueue),
# then we FIXME: probably need to scan it
if nzb.totalBytes == 0 and (len(Hellanzb.queue.currentNZBs()) or not writeQueue):
from Hellanzb.NZBLeecher.NZBParser import NZBTotalBytesParser
reactor.callInThread(NZBTotalBytesParser.getBytes, nzb)
else:
try:
shutil.move(nzbFile, Hellanzb.TEMP_DIR + os.sep)
except (IOError, OSError), e:
error("Unable to move invalid NZB: %s out of the way" % nzbFile)
debug("Unable to move invalid NZB: %s out of the way" % nzbFile, e)
def scanQueueDir(firstRun=False, justScan=False):
""" Find new/resume old NZB download sessions """
# t = time.time()
from Hellanzb.NZBLeecher.NZBModel import NZB
currentNZBs = []
for file in os.listdir(Hellanzb.CURRENT_DIR):
if Hellanzb.NZB_FILE_RE.search(file):
currentNZBs.append(os.path.join(Hellanzb.CURRENT_DIR, file))
# See if we're resuming a nzb fetch
resuming = False
displayNotification = False
newNZBs = []
queuedMap = {}
for nzb in Hellanzb.nzbQueue:
queuedMap[os.path.normpath(nzb.nzbFileName)] = nzb
for file in os.listdir(Hellanzb.QUEUE_DIR):
if file in Hellanzb.queueDirIgnore:
continue
if Hellanzb.NZB_FILE_RE.search(file):
if os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file)) not in queuedMap:
# Delay enqueueing recently modified NZBs
if not isOldEnough(os.path.join(Hellanzb.QUEUE_DIR, file)):
continue
newNZBs.append(os.path.join(Hellanzb.QUEUE_DIR, file))
elif os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file)) in queuedMap:
queuedMap.pop(os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file)))
elif isOldEnough(os.path.join(Hellanzb.QUEUE_DIR, file)):
if not nzbZipSearch(file) and not nzbGzipSearch(file):
Hellanzb.queueDirIgnore.append(file)
# Remove anything no longer in the queue directory
for nzb in queuedMap.itervalues():
Hellanzb.nzbQueue.remove(nzb)
if firstRun:
# enqueueNZBs() will delete the recovered state. Save it beforehand for sorting
queuedRecoveredState = Hellanzb.recoveredState.queued.copy()
enqueueNZBs(newNZBs, writeQueue=not firstRun)
if firstRun:
sortQueueFromRecoveredState(queuedRecoveredState)
# e = time.time() - t
if justScan:
# Done scanning -- don't bother loading a new NZB
# debug('Ziplick scanQueueDir (justScan): ' + Hellanzb.QUEUE_DIR + ' TOOK: ' + str(e))
# debug('Ziplick scanQueueDir (justScan): ' + Hellanzb.QUEUE_DIR)
Hellanzb.downloadScannerID = reactor.callLater(7, scanQueueDir, False, True)
return
# else:
# debug('Ziplick scanQueueDir: ' + Hellanzb.QUEUE_DIR)
if not currentNZBs:
if not Hellanzb.nzbQueue:
if firstRun:
writeStateXML()
# Nothing to do, lets wait 5 seconds and start over
Hellanzb.downloadScannerID = reactor.callLater(5, scanQueueDir)
if not firstRun and not justScan and not Hellanzb.loggedIdleMessage:
notify("Queue", "hellanzb", "No more nzbs left to download", False)
Hellanzb.loggedIdleMessage = True
return
# Start the next download
nzb = Hellanzb.nzbQueue[0]
nzbfilename = os.path.basename(nzb.nzbFileName)
del Hellanzb.nzbQueue[0]
# nzbfile will always be a absolute filename
nzbfile = os.path.join(Hellanzb.QUEUE_DIR, nzbfilename)
move(nzbfile, Hellanzb.CURRENT_DIR)
if not (len(newNZBs) == 1 and len(Hellanzb.nzbQueue) == 0):
# Show what's going to be downloaded next, unless the queue was empty, and we
# only found one nzb (The 'Found new nzb' message is enough in that case)
displayNotification = True
else:
# Resume the NZB in the CURRENT_DIR
nzbfilename = currentNZBs[0]
nzb = NZB.fromStateXML("downloading", nzbfilename)
nzbfilename = os.path.basename(nzb.nzbFileName)
displayNotification = True
del currentNZBs[0]
resuming = True
nzbfile = os.path.join(Hellanzb.CURRENT_DIR, nzbfilename)
nzb.nzbFileName = nzbfile
if firstRun:
writeStateXML()
if resuming:
if nzb.isParRecovery:
msg = "Resuming par recovery download"
else:
msg = "Resuming"
parseNZB(nzb, msg)
elif displayNotification:
parseNZB(nzb)
else:
parseNZB(nzb, quiet=True)
def enqueueNZBs(nzbFileOrFiles, next=False, writeQueue=True, category=None):
""" add one or a list of nzb files to the end of the queue """
if isinstance(nzbFileOrFiles, list) or isinstance(nzbFileOrFiles, tuple):
newNzbFiles = nzbFileOrFiles
else:
newNzbFiles = [nzbFileOrFiles]
if len(newNzbFiles) == 0:
return False
for nzbFile in newNzbFiles:
if validNZB(nzbFile):
if os.path.normpath(os.path.dirname(nzbFile)) != os.path.normpath(
Hellanzb.QUEUE_DIR):
copy(
nzbFile,
os.path.join(Hellanzb.QUEUE_DIR,
os.path.basename(nzbFile)))
nzbFile = os.path.join(Hellanzb.QUEUE_DIR,
os.path.basename(nzbFile))
found = False
for n in Hellanzb.nzbQueue:
if os.path.normpath(
n.nzbFileName) == os.path.normpath(nzbFile):
found = True
error('Unable to add nzb file to queue: ' + os.path.basename(nzbFile) + \
' it already exists!')
if found:
continue
from Hellanzb.NZBLeecher.NZBModel import NZB
name = os.path.basename(nzbFile)
nzb = NZB.fromStateXML('queued', nzbFile)
logMsg = msg = 'Found new nzb'
extraLog = []
if not next:
Hellanzb.nzbQueue.append(nzb)
else:
Hellanzb.nzbQueue.insert(0, nzb)
if nzb.msgid is not None:
extraLog.append('msgid: %s' % nzb.msgid)
if category:
nzb.category = category
extraLog.append('category: %s' % category)
if extraLog:
logMsg += ' (%s)' % ', '.join(extraLog)
logMsg += ': '
msg += ': '
info(logMsg + nzb.archiveName)
notify('Queue', 'hellanzb ' + msg, nzb.archiveName, False)
# Determine the total bytes of the NZB if it's not already
# known. If there's no NZBs in the queue (we're about to parse the
# NZB and begin downloading), there's no point in parsing
# now. Unless it's the first run of scanQueue (not writeQueue),
# then we FIXME: probably need to scan it
if nzb.totalBytes == 0 and (len(Hellanzb.queue.currentNZBs()) or \
not writeQueue):
from Hellanzb.NZBLeecher.NZBParser import NZBTotalBytesParser
reactor.callInThread(NZBTotalBytesParser.getBytes, nzb)
else:
try:
shutil.move(nzbFile, Hellanzb.TEMP_DIR + os.sep)
except (IOError, OSError), e:
error('Unable to move invalid NZB: %s out of the way' %
nzbFile)
debug(
'Unable to move invalid NZB: %s out of the way' % nzbFile,
e)
def scanQueueDir(firstRun=False, justScan=False):
""" Find new/resume old NZB download sessions """
#t = time.time()
from Hellanzb.NZBLeecher.NZBModel import NZB
currentNZBs = []
for file in os.listdir(Hellanzb.CURRENT_DIR):
if Hellanzb.NZB_FILE_RE.search(file):
currentNZBs.append(os.path.join(Hellanzb.CURRENT_DIR, file))
# See if we're resuming a nzb fetch
resuming = False
displayNotification = False
newNZBs = []
queuedMap = {}
for nzb in Hellanzb.nzbQueue:
queuedMap[os.path.normpath(nzb.nzbFileName)] = nzb
for file in os.listdir(Hellanzb.QUEUE_DIR):
if file in Hellanzb.queueDirIgnore:
continue
if Hellanzb.NZB_FILE_RE.search(file):
if os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR,
file)) not in queuedMap:
# Delay enqueueing recently modified NZBs
if not isOldEnough(os.path.join(Hellanzb.QUEUE_DIR, file)):
continue
newNZBs.append(os.path.join(Hellanzb.QUEUE_DIR, file))
elif os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR,
file)) in queuedMap:
queuedMap.pop(
os.path.normpath(os.path.join(Hellanzb.QUEUE_DIR, file)))
elif isOldEnough(os.path.join(Hellanzb.QUEUE_DIR, file)):
if not nzbZipSearch(file) and not nzbGzipSearch(file):
Hellanzb.queueDirIgnore.append(file)
# Remove anything no longer in the queue directory
for nzb in queuedMap.itervalues():
Hellanzb.nzbQueue.remove(nzb)
if firstRun:
# enqueueNZBs() will delete the recovered state. Save it beforehand for sorting
queuedRecoveredState = Hellanzb.recoveredState.queued.copy()
enqueueNZBs(newNZBs, writeQueue=not firstRun)
if firstRun:
sortQueueFromRecoveredState(queuedRecoveredState)
#e = time.time() - t
if justScan:
# Done scanning -- don't bother loading a new NZB
#debug('Ziplick scanQueueDir (justScan): ' + Hellanzb.QUEUE_DIR + ' TOOK: ' + str(e))
#debug('Ziplick scanQueueDir (justScan): ' + Hellanzb.QUEUE_DIR)
Hellanzb.downloadScannerID = reactor.callLater(7, scanQueueDir, False,
True)
return
#else:
# debug('Ziplick scanQueueDir: ' + Hellanzb.QUEUE_DIR)
if not currentNZBs:
if not Hellanzb.nzbQueue:
if firstRun:
writeStateXML()
# Nothing to do, lets wait 5 seconds and start over
Hellanzb.downloadScannerID = reactor.callLater(5, scanQueueDir)
if not firstRun and not justScan and not Hellanzb.loggedIdleMessage:
notify('Queue', 'hellanzb', 'No more nzbs left to download',
False)
Hellanzb.loggedIdleMessage = True
return
# Start the next download
nzb = Hellanzb.nzbQueue[0]
nzbfilename = os.path.basename(nzb.nzbFileName)
del Hellanzb.nzbQueue[0]
# nzbfile will always be a absolute filename
nzbfile = os.path.join(Hellanzb.QUEUE_DIR, nzbfilename)
move(nzbfile, Hellanzb.CURRENT_DIR)
if not (len(newNZBs) == 1 and len(Hellanzb.nzbQueue) == 0):
# Show what's going to be downloaded next, unless the queue was empty, and we
# only found one nzb (The 'Found new nzb' message is enough in that case)
displayNotification = True
else:
# Resume the NZB in the CURRENT_DIR
nzbfilename = currentNZBs[0]
nzb = NZB.fromStateXML('downloading', nzbfilename)
nzbfilename = os.path.basename(nzb.nzbFileName)
displayNotification = True
del currentNZBs[0]
resuming = True
nzbfile = os.path.join(Hellanzb.CURRENT_DIR, nzbfilename)
nzb.nzbFileName = nzbfile
if firstRun:
writeStateXML()
if resuming:
if nzb.isParRecovery:
msg = 'Resuming par recovery download'
else:
msg = 'Resuming'
parseNZB(nzb, msg)
elif displayNotification:
parseNZB(nzb)
else:
parseNZB(nzb, quiet=True)