def run(self):
import sabnzbd.nzbqueue
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf = job
if nzf:
sabnzbd.CheckFreeSpace()
filename = sanitize_filename(nzf.filename)
nzf.filename = filename
dupe = nzo.check_for_dupe(nzf)
filepath = get_filepath(cfg.download_dir.get_path(), nzo, filename)
if filepath:
logging.info('Decoding %s %s', filepath, nzf.type)
try:
filepath = _assemble(nzf, filepath, dupe)
except IOError, (errno, strerror):
# 28 == disk full => pause downloader
if errno == 28:
logging.error(Ta('Disk full! Forcing Pause'))
else:
logging.error(Ta('Disk error on creating file %s'), latin1(filepath))
# Pause without saving
sabnzbd.downloader.Downloader.do.pause(save=False)
except:
logging.error('Fatal error in Assembler', exc_info = True)
break
def run(self):
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf = job
if nzf:
# Check if enough disk space is free, if not pause downloader and send email
if diskspace(force=True)['download_dir'][1] < (
cfg.download_free.get_float() + nzf.bytes) / GIGI:
# Only warn and email once
if not sabnzbd.downloader.Downloader.do.paused:
logging.warning(
T('Too little diskspace forcing PAUSE'))
# Pause downloader, but don't save, since the disk is almost full!
sabnzbd.downloader.Downloader.do.pause()
sabnzbd.emailer.diskfull()
# Abort all direct unpackers, just to be sure
sabnzbd.directunpacker.abort_all()
# Place job back in queue and wait 30 seconds to hope it gets resolved
self.process(job)
sleep(30)
continue
# Prepare filename
nzo.verify_nzf_filename(nzf)
nzf.filename = sanitize_filename(nzf.filename)
filepath = get_filepath(long_path(cfg.download_dir.get_path()),
nzo, nzf.filename)
nzf.filename = get_filename(filepath)
if filepath:
logging.info('Decoding %s %s', filepath, nzf.type)
try:
filepath = self.assemble(nzf, filepath)
except IOError, (errno, strerror):
# If job was deleted or in active post-processing, ignore error
if not nzo.deleted and not nzo.is_gone(
) and not nzo.pp_active:
# 28 == disk full => pause downloader
if errno == 28:
logging.error(T('Disk full! Forcing Pause'))
else:
logging.error(
T('Disk error on creating file %s'),
clip_path(filepath))
# Log traceback
logging.info('Traceback: ', exc_info=True)
# Pause without saving
sabnzbd.downloader.Downloader.do.pause()
continue
except:
logging.error(T('Fatal error in Assembler'),
exc_info=True)
break
def run(self):
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf = job
if nzf:
# Check if enough disk space is free, if not pause downloader and send email
if diskspace(force=True)['download_dir'][1] < (cfg.download_free.get_float() + nzf.bytes) / GIGI:
# Only warn and email once
if not sabnzbd.downloader.Downloader.do.paused:
logging.warning(T('Too little diskspace forcing PAUSE'))
# Pause downloader, but don't save, since the disk is almost full!
sabnzbd.downloader.Downloader.do.pause()
sabnzbd.emailer.diskfull()
# Abort all direct unpackers, just to be sure
sabnzbd.directunpacker.abort_all()
# Place job back in queue and wait 30 seconds to hope it gets resolved
self.process(job)
sleep(30)
continue
# Prepare filename
nzo.verify_nzf_filename(nzf)
nzf.filename = sanitize_filename(nzf.filename)
filepath = get_filepath(long_path(cfg.download_dir.get_path()), nzo, nzf.filename)
nzf.filename = get_filename(filepath)
if filepath:
logging.info('Decoding %s %s', filepath, nzf.type)
try:
filepath = self.assemble(nzf, filepath)
except IOError, (errno, strerror):
# If job was deleted or in active post-processing, ignore error
if not nzo.deleted and not nzo.is_gone() and not nzo.pp_active:
# 28 == disk full => pause downloader
if errno == 28:
logging.error(T('Disk full! Forcing Pause'))
else:
logging.error(T('Disk error on creating file %s'), clip_path(filepath))
# Log traceback
logging.info('Traceback: ', exc_info=True)
# Pause without saving
sabnzbd.downloader.Downloader.do.pause()
continue
except:
logging.error(T('Fatal error in Assembler'), exc_info=True)
break
def run(self):
import sabnzbd.nzbqueue
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf = job
if nzf:
sabnzbd.CheckFreeSpace()
filename = sanitize_filename(nzf.filename)
nzf.filename = filename
dupe = nzo.check_for_dupe(nzf)
filepath = get_filepath(cfg.download_dir.get_path(), nzo,
filename)
if filepath:
logging.info('Decoding %s %s', filepath, nzf.type)
try:
filepath = _assemble(nzf, filepath, dupe)
except IOError, (errno, strerror):
if nzo.deleted:
# Job was deleted, ignore error
pass
else:
# 28 == disk full => pause downloader
if errno == 28:
logging.error(Ta('Disk full! Forcing Pause'))
else:
logging.error(
Ta('Disk error on creating file %s'),
latin1(filepath))
# Pause without saving
sabnzbd.downloader.Downloader.do.pause(save=False)
except:
logging.error('Fatal error in Assembler',
exc_info=True)
break
def run(self):
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf = job
if nzf:
sabnzbd.CheckFreeSpace()
# We allow win_devices because otherwise par2cmdline fails to repair
filename = sanitize_filename(nzf.filename,
allow_win_devices=True)
nzf.filename = filename
dupe = nzo.check_for_dupe(nzf)
filepath = get_filepath(long_path(cfg.download_dir.get_path()),
nzo, filename)
if filepath:
logging.info('Decoding %s %s', filepath, nzf.type)
try:
filepath = _assemble(nzf, filepath, dupe)
except IOError, (errno, strerror):
# If job was deleted, ignore error
if not nzo.is_gone():
# 28 == disk full => pause downloader
if errno == 28:
logging.error(T('Disk full! Forcing Pause'))
else:
logging.error(
T('Disk error on creating file %s'),
clip_path(filepath))
# Pause without saving
sabnzbd.downloader.Downloader.do.pause(save=False)
continue
except:
logging.error(T('Fatal error in Assembler'),
exc_info=True)
break
def run(self):
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf = job
if nzf:
sabnzbd.CheckFreeSpace()
# We allow win_devices because otherwise par2cmdline fails to repair
filename = sanitize_filename(nzf.filename, allow_win_devices=True)
nzf.filename = filename
dupe = nzo.check_for_dupe(nzf)
filepath = get_filepath(long_path(cfg.download_dir.get_path()), nzo, filename)
if filepath:
logging.info('Decoding %s %s', filepath, nzf.type)
try:
filepath = _assemble(nzf, filepath, dupe)
except IOError, (errno, strerror):
# If job was deleted, ignore error
if not nzo.is_gone():
# 28 == disk full => pause downloader
if errno == 28:
logging.error(T('Disk full! Forcing Pause'))
else:
logging.error(T('Disk error on creating file %s'), clip_path(filepath))
# Pause without saving
sabnzbd.downloader.Downloader.do.pause(save=False)
continue
except:
logging.error(T('Fatal error in Assembler'), exc_info=True)
break
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo:
# Re-queue when too early and still active
if future_nzo.wait and future_nzo.wait > time.time():
self.add(url, future_nzo)
time.sleep(1.0)
continue
# Paused
if future_nzo.status == Status.PAUSED:
self.add(url, future_nzo)
time.sleep(1.0)
continue
url = url.replace(' ', '')
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing', url)
continue
filename = None
category = None
gzipped = False
nzo_info = {}
wait = 0
retry = True
fn = None
logging.info('Grabbing URL %s', url)
try:
fn = _build_request(url)
except Exception, e:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s', error1, url)
if 'certificate_verify_failed' in error1 or 'certificateerror' in error0:
msg = T('Server %s uses an untrusted HTTPS certificate') % ''
msg += ' - https://sabnzbd.org/certificate-errors'
retry = False
elif 'nodename nor servname provided' in error1:
msg = T('Server name does not resolve')
retry = False
elif '401' in error1 or 'unauthorized' in error1:
msg = T('Unauthorized access')
retry = False
elif '404' in error1:
msg = T('File not on server')
retry = False
elif hasattr(e, 'headers') and 'retry-after' in e.headers:
# Catch if the server send retry (e.headers is case-INsensitive)
wait = misc.int_conv(e.headers['retry-after'])
new_url = dereferring(url, fn)
if new_url:
self.add(new_url, future_nzo)
continue
if fn:
for hdr in fn.headers:
try:
item = hdr.lower()
value = fn.headers[hdr]
except:
continue
if item in ('content-encoding',) and value == 'gzip':
gzipped = True
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo',):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name',):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item == 'x-dnzb-password':
nzo_info['password'] = value
elif item == 'retry-after':
wait = misc.int_conv(value)
# Rating fields
if item in _RARTING_FIELDS:
nzo_info[item] = value
if not filename and "filename=" in value:
filename = value[value.index("filename=") + 9:].strip(';').strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fn = None
elif retry:
fn, msg, retry, wait, data = _analyse(fn, url)
if not fn:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
bad_fetch(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(url)
elif '&nzbname=' in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find('&nzbname=') + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if gzipped:
filename += '.gz'
if not data:
try:
data = fn.read()
except (IncompleteRead, IOError):
bad_fetch(future_nzo, url, T('Server could not complete request'))
fn.close()
continue
fn.close()
if '<nzb' in data and misc.get_ext(filename) != '.nzb':
filename += '.nzb'
# Sanitize filename first (also removing forbidden Windows-names)
filename = misc.sanitize_filename(filename)
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)
path = os.path.join(path, filename)
f = open(path, 'wb')
f.write(data)
f.close()
del data
# Check if nzb file
if misc.get_ext(filename) in ('.nzb', '.gz', 'bz2'):
res = dirscanner.ProcessSingleFile(filename, path, pp=pp, script=script, cat=cat, priority=priority,
nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)[0]
if res:
if res == -2:
logging.info('Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False)
continue
else:
logging.info('Unknown error fetching NZB, retry after 2 min %s', url)
when = 120
self.add(url, future_nzo, when)
# Check if a supported archive
else:
status, zf, exp_ext = dirscanner.is_archive(path)
if status == 0:
if misc.get_ext(filename) not in ('.rar', '.zip', '.7z'):
filename = filename + exp_ext
os.rename(path, path + exp_ext)
path = path + exp_ext
dirscanner.ProcessArchiveFile(filename, path, pp, script, cat, priority=priority,
nzbname=nzbname, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fn)
except:
pass
logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url)
self.add(url, future_nzo, 30)
except:
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo:
# Re-queue when too early and still active
if future_nzo.url_wait and future_nzo.url_wait > time.time():
self.add(url, future_nzo)
time.sleep(1.0)
continue
# Paused
if future_nzo.status == Status.PAUSED:
self.add(url, future_nzo)
time.sleep(1.0)
continue
url = url.replace(' ', '')
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing', url)
continue
filename = None
category = None
gzipped = False
nzo_info = {}
wait = 0
retry = True
fetch_request = None
logging.info('Grabbing URL %s', url)
try:
fetch_request = _build_request(url)
except Exception, e:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s', error1, url)
if 'certificate_verify_failed' in error1 or 'certificateerror' in error0:
msg = T('Server %s uses an untrusted HTTPS certificate') % ''
msg += ' - https://sabnzbd.org/certificate-errors'
retry = False
elif 'nodename nor servname provided' in error1:
msg = T('Server name does not resolve')
retry = False
elif '401' in error1 or 'unauthorized' in error1:
msg = T('Unauthorized access')
retry = False
elif '404' in error1:
msg = T('File not on server')
retry = False
elif hasattr(e, 'headers') and 'retry-after' in e.headers:
# Catch if the server send retry (e.headers is case-INsensitive)
wait = misc.int_conv(e.headers['retry-after'])
# Check if dereference is used
new_url = dereferring(url, fetch_request)
if new_url:
self.add(new_url, future_nzo)
continue
if fetch_request:
for hdr in fetch_request.headers:
try:
item = hdr.lower()
value = fetch_request.headers[hdr]
except:
continue
if item in ('content-encoding',) and value == 'gzip':
gzipped = True
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo',):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name',):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item == 'x-dnzb-password':
nzo_info['password'] = value
elif item == 'retry-after':
wait = misc.int_conv(value)
# Rating fields
if item in _RARTING_FIELDS:
nzo_info[item] = value
# Get filename from Content-Disposition header
if not filename and "filename=" in value:
filename = value[value.index("filename=") + 9:].strip(';').strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fetch_request = None
elif retry:
fetch_request, msg, retry, wait, data = _analyse(fetch_request, future_nzo)
if not fetch_request:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
self.fail_to_history(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(urllib2.unquote(url))
# URL was redirected, maybe the redirect has better filename?
# Check if the original URL has extension
if url != fetch_request.url and misc.get_ext(filename) not in VALID_NZB_FILES:
filename = os.path.basename(urllib2.unquote(fetch_request.url))
elif '&nzbname=' in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find('&nzbname=') + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if gzipped:
filename += '.gz'
if not data:
try:
data = fetch_request.read()
except (IncompleteRead, IOError):
self.fail_to_history(future_nzo, url, T('Server could not complete request'))
fetch_request.close()
continue
fetch_request.close()
if '<nzb' in data and misc.get_ext(filename) != '.nzb':
filename += '.nzb'
# Sanitize filename first (also removing forbidden Windows-names)
filename = misc.sanitize_filename(filename)
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)
path = os.path.join(path, filename)
f = open(path, 'wb')
f.write(data)
f.close()
del data
# Check if nzb file
if misc.get_ext(filename) in VALID_NZB_FILES:
res = dirscanner.ProcessSingleFile(filename, path, pp=pp, script=script, cat=cat, priority=priority,
nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)[0]
if res:
if res == -2:
logging.info('Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False)
continue
else:
logging.info('Unknown error fetching NZB, retry after 2 min %s', url)
when = 120
self.add(url, future_nzo, when)
else:
# Check if a supported archive
status, zf, exp_ext = dirscanner.is_archive(path)
if status == 0:
if misc.get_ext(filename) not in ('.rar', '.zip', '.7z'):
filename = filename + exp_ext
os.rename(path, path + exp_ext)
path = path + exp_ext
dirscanner.ProcessArchiveFile(filename, path, pp, script, cat, priority=priority,
nzbname=nzbname, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)
else:
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fetch_request)
except:
pass
logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url)
self.add(url, future_nzo, 30)
except:
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
# Don't pound the website!
time.sleep(5.0)
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo and future_nzo.wait and future_nzo.wait > time.time():
# Re-queue when too early and still active
self.add(url, future_nzo)
continue
url = url.replace(' ', '')
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing', url)
continue
logging.info('Grabbing URL %s', url)
req = urllib2.Request(url)
req.add_header('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__)
if not [True for item in _BAD_GZ_HOSTS if item in url]:
req.add_header('Accept-encoding', 'gzip')
filename = None
category = None
gzipped = False
nzo_info = {}
wait = 0
retry = True
fn = None
try:
fn = urllib2.urlopen(req)
except:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s', error1, url)
if 'certificate_verify_failed' in error1 or 'certificateerror' in error0:
msg = T('Server %s uses an untrusted HTTPS certificate') % ''
retry = False
elif 'nodename nor servname provided' in error1:
msg = T('Server name does not resolve')
retry = False
elif '401' in error1 or 'unauthorized' in error1:
msg = T('Unauthorized access')
retry = False
elif '404' in error1:
msg = T('File not on server')
retry = False
new_url = dereferring(url, fn)
if new_url:
self.add(new_url, future_nzo)
continue
if fn:
for hdr in fn.headers:
try:
item = hdr.lower()
value = fn.headers[hdr]
except:
continue
if item in ('content-encoding',) and value == 'gzip':
gzipped = True
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo',):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name',):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item == 'retry-after':
# For NZBFinder
wait = misc.int_conv(value)
if not filename and "filename=" in value:
filename = value[value.index("filename=") + 9:].strip(';').strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fn = None
elif retry:
fn, msg, retry, wait, data = _analyse(fn, url)
if not fn:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
bad_fetch(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(url) + '.nzb'
elif '&nzbname=' in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find('&nzbname=') + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if gzipped:
filename = filename + '.gz'
if not data:
data = fn.read()
fn.close()
# Sanatize filename first
filename = misc.sanitize_filename(filename)
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)
path = os.path.join(path, filename)
f = open(path, 'wb')
f.write(data)
f.close()
del data
# Check if nzb file
if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz', 'bz2'):
res = dirscanner.ProcessSingleFile(filename, path, pp=pp, script=script, cat=cat, priority=priority,
nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)[0]
if res:
if res == -2:
logging.info('Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False)
continue
else:
logging.info('Unknown error fetching NZB, retry after 2 min %s', url)
when = 120
self.add(url, future_nzo, when)
# Check if a supported archive
else:
if dirscanner.ProcessArchiveFile(filename, path, pp, script, cat, priority=priority,
nzbname=nzbname, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)[0]:
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fn)
except:
pass
logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url)
self.add(url, future_nzo, 30)
except:
logging.error(T('URLGRABBER CRASHED'), exc_info=True)
logging.debug("URLGRABBER Traceback: ", exc_info=True)
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
# Don't pound the website!
time.sleep(5.0)
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo and future_nzo.wait and future_nzo.wait > time.time(
):
# Re-queue when too early and still active
self.add(url, future_nzo)
continue
url = url.replace(' ', '')
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing',
url)
continue
logging.info('Grabbing URL %s', url)
req = urllib2.Request(url)
req.add_header('User-Agent',
'SABnzbd+/%s' % sabnzbd.version.__version__)
if not [True for item in _BAD_GZ_HOSTS if item in url]:
req.add_header('Accept-encoding', 'gzip')
filename = None
category = None
gzipped = False
nzo_info = {}
wait = 0
retry = True
fn = None
try:
fn = urllib2.urlopen(req)
except:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s',
error1, url)
if 'certificate_verify_failed' in error1 or 'certificateerror' in error0:
msg = T('Server %s uses an untrusted HTTPS certificate'
) % ''
retry = False
elif 'nodename nor servname provided' in error1:
msg = T('Server name does not resolve')
retry = False
elif '401' in error1 or 'unauthorized' in error1:
msg = T('Unauthorized access')
retry = False
elif '404' in error1:
msg = T('File not on server')
retry = False
new_url = dereferring(url, fn)
if new_url:
self.add(new_url, future_nzo)
continue
if fn:
for hdr in fn.headers:
try:
item = hdr.lower()
value = fn.headers[hdr]
except:
continue
if item in ('content-encoding', ) and value == 'gzip':
gzipped = True
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo', ):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name', ):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item == 'x-dnzb-password':
nzo_info['password'] = value
elif item == 'retry-after':
# For NZBFinder
wait = misc.int_conv(value)
if not filename and "filename=" in value:
filename = value[value.index("filename=") +
9:].strip(';').strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fn = None
elif retry:
fn, msg, retry, wait, data = _analyse(fn, url)
if not fn:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
bad_fetch(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(url) + '.nzb'
elif '&nzbname=' in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find('&nzbname=') + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if gzipped:
filename = filename + '.gz'
if not data:
data = fn.read()
fn.close()
# Sanatize filename first
filename = misc.sanitize_filename(filename)
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)
path = os.path.join(path, filename)
f = open(path, 'wb')
f.write(data)
f.close()
del data
# Check if nzb file
if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz',
'bz2'):
res = dirscanner.ProcessSingleFile(
filename,
path,
pp=pp,
script=script,
cat=cat,
priority=priority,
nzbname=nzbname,
nzo_info=nzo_info,
url=future_nzo.url,
keep=False,
nzo_id=future_nzo.nzo_id)[0]
if res:
if res == -2:
logging.info(
'Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id,
add_to_history=False)
continue
else:
logging.info(
'Unknown error fetching NZB, retry after 2 min %s',
url)
when = 120
self.add(url, future_nzo, when)
# Check if a supported archive
else:
if dirscanner.ProcessArchiveFile(
filename,
path,
pp,
script,
cat,
priority=priority,
nzbname=nzbname,
url=future_nzo.url,
keep=False,
nzo_id=future_nzo.nzo_id)[0]:
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fn)
except:
pass
logging.info(
'Unknown filetype when fetching NZB, retry after 30s %s',
url)
self.add(url, future_nzo, 30)
except:
logging.error(T('URLGRABBER CRASHED'), exc_info=True)
logging.debug("URLGRABBER Traceback: ", exc_info=True)