def listQueue(includeIds=True, convertToUnicode=True):
""" Return a listing of the current queue. By default this function will convert all
strings to unicode, as it's only used right now for the return of XMLRPC calls """
members = []
for nzb in Hellanzb.nzbQueue:
if includeIds:
name = archiveName(os.path.basename(nzb.nzbFileName))
rarPassword = nzb.rarPassword
if convertToUnicode:
name = toUnicode(name)
rarPassword = toUnicode(rarPassword)
member = {
'id': nzb.id,
'nzbName': name,
'is_par_recovery': nzb.isParRecovery
}
if rarPassword is not None:
member['rarPassword'] = rarPassword
if nzb.msgid is not None:
member['msgid'] = nzb.msgid
if not nzb.calculatingBytes:
member['total_mb'] = nzb.totalBytes / 1024 / 1024
else:
member = os.path.basename(nzb.nzbFileName)
members.append(member)
return members
def listQueue(includeIds=True, convertToUnicode=True):
""" Return a listing of the current queue. By default this function will convert all
strings to unicode, as it's only used right now for the return of XMLRPC calls """
members = []
for nzb in Hellanzb.nzbQueue:
if includeIds:
name = archiveName(os.path.basename(nzb.nzbFileName))
rarPassword = nzb.rarPassword
if convertToUnicode:
name = toUnicode(name)
rarPassword = toUnicode(rarPassword)
member = {"id": nzb.id, "nzbName": name, "is_par_recovery": nzb.isParRecovery}
if rarPassword is not None:
member["rarPassword"] = rarPassword
if nzb.msgid is not None:
member["msgid"] = nzb.msgid
if not nzb.calculatingBytes:
member["total_mb"] = nzb.totalBytes / 1024 / 1024
else:
member = os.path.basename(nzb.nzbFileName)
members.append(member)
return members
def findAndLoadPostponedDir(nzb):
""" Move a postponed working directory for the specified nzb, if one is found, to the
WORKING_DIR """
def fixNZBFileName(nzb):
if os.path.normpath(os.path.dirname(nzb.destDir)) == os.path.normpath(
Hellanzb.POSTPONED_DIR):
nzb.destDir = Hellanzb.WORKING_DIR
nzbfilename = nzb.nzbFileName
d = os.path.join(Hellanzb.POSTPONED_DIR, archiveName(nzbfilename))
if os.path.isdir(d):
try:
os.rmdir(Hellanzb.WORKING_DIR)
except OSError:
files = os.listdir(Hellanzb.WORKING_DIR)[0]
if len(files):
name = files[0]
ext = getFileExtension(name)
if ext != None:
name = name.replace(ext, '')
move(Hellanzb.WORKING_DIR,
os.path.join(Hellanzb.TEMP_DIR, name))
else:
debug('ERROR Stray WORKING_DIR!: ' +
str(os.listdir(Hellanzb.WORKING_DIR)))
name = os.path.join(Hellanzb.TEMP_DIR, 'stray_WORKING_DIR')
hellaRename(name)
move(Hellanzb.WORKING_DIR, name)
move(d, Hellanzb.WORKING_DIR)
Hellanzb.queue.unpostpone(nzb)
ensureSafePostponedLoad(nzb.nzbFileName)
info('Loaded postponed directory: ' + archiveName(nzbfilename))
fixNZBFileName(nzb)
return True
else:
fixNZBFileName(nzb)
return False
def findAndLoadPostponedDir(nzb):
""" Move a postponed working directory for the specified nzb, if one is found, to the
WORKING_DIR """
def fixNZBFileName(nzb):
if os.path.normpath(os.path.dirname(nzb.destDir)) == os.path.normpath(Hellanzb.POSTPONED_DIR):
nzb.destDir = Hellanzb.WORKING_DIR
nzbfilename = nzb.nzbFileName
d = os.path.join(Hellanzb.POSTPONED_DIR, archiveName(nzbfilename))
if os.path.isdir(d):
try:
os.rmdir(Hellanzb.WORKING_DIR)
except OSError:
files = os.listdir(Hellanzb.WORKING_DIR)[0]
if len(files):
name = files[0]
ext = getFileExtension(name)
if ext != None:
name = name.replace(ext, "")
move(Hellanzb.WORKING_DIR, os.path.join(Hellanzb.TEMP_DIR, name))
else:
debug("ERROR Stray WORKING_DIR!: " + str(os.listdir(Hellanzb.WORKING_DIR)))
name = os.path.join(Hellanzb.TEMP_DIR, "stray_WORKING_DIR")
hellaRename(name)
move(Hellanzb.WORKING_DIR, name)
move(d, Hellanzb.WORKING_DIR)
Hellanzb.queue.unpostpone(nzb)
ensureSafePostponedLoad(nzb.nzbFileName)
info("Loaded postponed directory: " + archiveName(nzbfilename))
fixNZBFileName(nzb)
return True
else:
fixNZBFileName(nzb)
return False
def unpostpone(self, nzb):
""" Recall a postponed NZB """
self.nzbFilesLock.acquire()
arName = archiveName(nzb.nzbFileName)
found = []
for nzbFile in self.postponedNzbFiles:
# FIXME:
# Why is this not nzbFile.nzb == nzb?
if nzbFile.nzb.archiveName == arName:
found.append(nzbFile)
for nzbFile in found:
self.postponedNzbFiles.remove(nzbFile)
self.nzbFilesLock.release()
def resumePostProcessors():
""" Pickup left off Post Processors that were cancelled via CTRL-C """
# FIXME: with the new queue, could kill the processing dir sym links (for windows)
from Hellanzb.NZBLeecher.NZBModel import NZB
for archiveDirName in os.listdir(Hellanzb.PROCESSING_DIR):
if archiveDirName[0] == '.':
continue
archive = NZB.fromStateXML('processing', archiveDirName)
troll = PostProcessor.PostProcessor(archive)
info('Resuming post processor: ' + archiveName(archiveDirName))
troll.start()
def unpostpone(self, nzb):
""" Recall a postponed NZB """
self.nzbFilesLock.acquire()
arName = archiveName(nzb.nzbFileName)
found = []
for nzbFile in self.postponedNzbFiles:
# FIXME:
# Why is this not nzbFile.nzb == nzb?
if nzbFile.nzb.archiveName == arName:
found.append(nzbFile)
for nzbFile in found:
self.postponedNzbFiles.remove(nzbFile)
self.nzbFilesLock.release()
def fromStateXML(type, target):
""" Factory method, returns a new NZB object for the specified target, and recovers
the NZB state from the RecoveredState object if the target exists there for
the specified type (such as 'processing', 'downloading') """
if type == 'processing':
recoveredDict = Hellanzb.recoveredState.getRecoveredDict(
type, target)
archiveDir = os.path.join(Hellanzb.PROCESSING_DIR, target)
if recoveredDict and recoveredDict.get('nzbFileName') is not None:
target = recoveredDict.get('nzbFileName')
else:
# If this is a processing recovery request, and we didn't recover any
# state information, we'll consider this a basic Archive object (it has no
# accompanying .NZB file to keep track of)
return Archive.fromStateXML(archiveDir, recoveredDict)
else:
recoveredDict = Hellanzb.recoveredState.getRecoveredDict(
type, archiveName(target))
# Pass the id in with the constructor (instead of setting it after the fact) --
# otherwise the constructor would unnecessarily incremenet the IDPool
nzbId = None
if recoveredDict:
nzbId = recoveredDict['id']
nzb = NZB(target, nzbId)
if type == 'processing':
nzb.archiveDir = archiveDir
if recoveredDict:
for key, value in recoveredDict.iteritems():
if key == 'id' or key == 'order':
continue
if key == 'neededBlocks':
value = int(value)
if key == 'totalBytes':
nzb.calculatingBytes = False
value = int(value)
if key == 'downloadTime':
value = float(value)
if key == 'parType':
value = getParEnum(value)
setattr(nzb, key, value)
return nzb
def fromStateXML(type, target):
""" Factory method, returns a new NZB object for the specified target, and recovers
the NZB state from the RecoveredState object if the target exists there for
the specified type (such as 'processing', 'downloading') """
if type == 'processing':
recoveredDict = Hellanzb.recoveredState.getRecoveredDict(type, target)
archiveDir = os.path.join(Hellanzb.PROCESSING_DIR, target)
if recoveredDict and recoveredDict.get('nzbFileName') is not None:
target = recoveredDict.get('nzbFileName')
else:
# If this is a processing recovery request, and we didn't recover any
# state information, we'll consider this a basic Archive object (it has no
# accompanying .NZB file to keep track of)
return Archive.fromStateXML(archiveDir, recoveredDict)
else:
recoveredDict = Hellanzb.recoveredState.getRecoveredDict(type,
archiveName(target))
# Pass the id in with the constructor (instead of setting it after the fact) --
# otherwise the constructor would unnecessarily incremenet the IDPool
nzbId = None
if recoveredDict:
nzbId = recoveredDict['id']
nzb = NZB(target, nzbId)
if type == 'processing':
nzb.archiveDir = archiveDir
if recoveredDict:
for key, value in recoveredDict.iteritems():
if key == 'id' or key == 'order':
continue
if key == 'neededBlocks':
value = int(value)
if key == 'totalBytes':
nzb.calculatingBytes = False
value = int(value)
if key == 'downloadTime':
value = float(value)
if key == 'parType':
value = getParEnum(value)
setattr(nzb, key, value)
return nzb
def __init__(self,
nzbFileName,
id=None,
rarPassword=None,
archiveDir=None,
category=''):
Archive.__init__(self, archiveDir, id, None, rarPassword)
## NZB file general information
self.nzbFileName = nzbFileName
self.archiveName = archiveName(self.nzbFileName) # pretty name
self.msgid = None
filename = os.path.basename(nzbFileName)
self.msgid = getMsgId(filename)
if self.msgid:
self.msgid = int(self.msgid)
self.nzbFiles = []
self.skippedParFiles = []
self.category = category
## Where the nzb files will be downloaded
self.destDir = Hellanzb.WORKING_DIR
## A cancelled NZB is marked for death. ArticleDecoder will dispose of any
## recently downloaded data that might have been downloading during the time the
## cancel call was made (after the fact cleanup)
self.canceled = False
self.canceledLock = Lock()
## Acquired during assembly of an NZBFile
self.assembleLock = Lock()
## Total bytes this NZB represents
self.totalBytes = 0
## Whether the total byte count of this NZB is still being calculated
self.calculatingBytes = True
## How many bytes were skipped for downloading
self.totalSkippedBytes = 0
## How many bytes have been downloaded for this NZB
self.totalReadBytes = 0
## How many bytes of encoded article data is currently cached to memory
self.cachedArticleDataBytes = 0
## Time this NZB began downloading
self.downloadStartTime = None
## Amount of time taken to download the NZB
self.downloadTime = 0
## Amount of time taken to download AND decode the NZB
self.downloadAndDecodeTime = 0
## Whether or not we should redownload NZBFile and NZBSegment files on disk that
## are 0 bytes in size
self.overwriteZeroByteFiles = True
# All segment0001s are downloaded first. Every time we successfully decode a
# segment0001, we add to this number
self.firstSegmentsDownloaded = 0
## Whether or not this NZB is downloading in par recovery mode
self.isParRecovery = False
## Whether or not this is an NZB that contains all par files
self.allParsMode = False
## Skipped par file's subjects are kept here, in a list, during post
## processing. This list is arranged by the file's size
self.skippedParSubjects = None
## The number of par blocks (or par files for par1 mode), queued to download
## recovery blocks, the par version, and the par prefix for the current par
## recovery download
self.neededBlocks = 0
self.queuedBlocks = 0
self.parType = None
self.parPrefix = None
def __init__(self, nzbFileName, id = None, rarPassword = None, archiveDir = None,
category = ''):
Archive.__init__(self, archiveDir, id, None, rarPassword)
## NZB file general information
self.nzbFileName = nzbFileName
self.archiveName = archiveName(self.nzbFileName) # pretty name
self.msgid = None
filename = os.path.basename(nzbFileName)
self.msgid = getMsgId(filename)
if self.msgid:
self.msgid = int(self.msgid)
self.nzbFiles = []
self.skippedParFiles = []
self.category = category
## Where the nzb files will be downloaded
self.destDir = Hellanzb.WORKING_DIR
## A cancelled NZB is marked for death. ArticleDecoder will dispose of any
## recently downloaded data that might have been downloading during the time the
## cancel call was made (after the fact cleanup)
self.canceled = False
self.canceledLock = Lock()
## Acquired during assembly of an NZBFile
self.assembleLock = Lock()
## Total bytes this NZB represents
self.totalBytes = 0
## Whether the total byte count of this NZB is still being calculated
self.calculatingBytes = True
## How many bytes were skipped for downloading
self.totalSkippedBytes = 0
## How many bytes have been downloaded for this NZB
self.totalReadBytes = 0
## How many bytes of encoded article data is currently cached to memory
self.cachedArticleDataBytes = 0
## Time this NZB began downloading
self.downloadStartTime = None
## Amount of time taken to download the NZB
self.downloadTime = 0
## Amount of time taken to download AND decode the NZB
self.downloadAndDecodeTime = 0
## Whether or not we should redownload NZBFile and NZBSegment files on disk that
## are 0 bytes in size
self.overwriteZeroByteFiles = True
# All segment0001s are downloaded first. Every time we successfully decode a
# segment0001, we add to this number
self.firstSegmentsDownloaded = 0
## Whether or not this NZB is downloading in par recovery mode
self.isParRecovery = False
## Whether or not this is an NZB that contains all par files
self.allParsMode = False
## Skipped par file's subjects are kept here, in a list, during post
## processing. This list is arranged by the file's size
self.skippedParSubjects = None
## The number of par blocks (or par files for par1 mode), queued to download
## recovery blocks, the par version, and the par prefix for the current par
## recovery download
self.neededBlocks = 0
self.queuedBlocks = 0
self.parType = None
self.parPrefix = None