def nzb_redirect(wdir, nzbname, pp, script, cat, priority):
""" Check if this job contains only NZB files,
if so send to queue and remove if on CleanList
Returns list of processed NZB's
"""
files = recursive_listdir(wdir)
for file_ in files:
if os.path.splitext(file_)[1].lower() != '.nzb':
return None
# For multiple NZBs, cannot use the current job name
if len(files) != 1:
nzbname = None
# Process all NZB files
for file_ in files:
dirscanner.ProcessSingleFile(os.path.split(file_)[1],
file_,
pp,
script,
cat,
priority=priority,
keep=False,
dup_check=False,
nzbname=nzbname)
return files
def nzb_redirect(wdir, nzbname, pp, script, cat, priority):
""" Check if this job contains only NZB files,
if so send to queue and remove if on CleanList
Returns list of processed NZB's
"""
lst = []
try:
files = os.listdir(wdir)
except:
files = []
for file_ in files:
if os.path.splitext(file_)[1].lower() != '.nzb':
return lst
# For a single NZB, use the current job name
if len(files) != 1:
nzbname = None
# Process all NZB files
for file_ in files:
if file_.lower().endswith('.nzb'):
dirscanner.ProcessSingleFile(file_,
os.path.join(wdir, file_),
pp,
script,
cat,
priority=priority,
keep=False,
dup_check=False,
nzbname=nzbname)
lst.append(file_)
return lst
def application_openFiles_(self, nsapp, filenames):
# logging.info('[osx] file open')
# logging.info('[osx] file : %s' % (filenames))
for name in filenames:
logging.info('[osx] receiving from OSX : %s', name)
if os.path.exists(name):
fn = get_filename(name)
# logging.info('[osx] filename : %s' % (fn))
if fn:
if get_ext(name) in VALID_ARCHIVES:
# logging.info('[osx] archive')
dirscanner.ProcessArchiveFile(fn, name, keep=True)
elif get_ext(name) in ('.nzb', '.gz', '.bz2'):
# logging.info('[osx] nzb')
dirscanner.ProcessSingleFile(fn, name, keep=True)
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo:
# Re-queue when too early and still active
if future_nzo.wait and future_nzo.wait > time.time():
self.add(url, future_nzo)
time.sleep(1.0)
continue
# Paused
if future_nzo.status == Status.PAUSED:
self.add(url, future_nzo)
time.sleep(1.0)
continue
url = url.replace(' ', '')
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing', url)
continue
filename = None
category = None
gzipped = False
nzo_info = {}
wait = 0
retry = True
fn = None
logging.info('Grabbing URL %s', url)
try:
fn = _build_request(url)
except Exception, e:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s', error1, url)
if 'certificate_verify_failed' in error1 or 'certificateerror' in error0:
msg = T('Server %s uses an untrusted HTTPS certificate') % ''
msg += ' - https://sabnzbd.org/certificate-errors'
retry = False
elif 'nodename nor servname provided' in error1:
msg = T('Server name does not resolve')
retry = False
elif '401' in error1 or 'unauthorized' in error1:
msg = T('Unauthorized access')
retry = False
elif '404' in error1:
msg = T('File not on server')
retry = False
elif hasattr(e, 'headers') and 'retry-after' in e.headers:
# Catch if the server send retry (e.headers is case-INsensitive)
wait = misc.int_conv(e.headers['retry-after'])
new_url = dereferring(url, fn)
if new_url:
self.add(new_url, future_nzo)
continue
if fn:
for hdr in fn.headers:
try:
item = hdr.lower()
value = fn.headers[hdr]
except:
continue
if item in ('content-encoding',) and value == 'gzip':
gzipped = True
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo',):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name',):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item == 'x-dnzb-password':
nzo_info['password'] = value
elif item == 'retry-after':
wait = misc.int_conv(value)
# Rating fields
if item in _RARTING_FIELDS:
nzo_info[item] = value
if not filename and "filename=" in value:
filename = value[value.index("filename=") + 9:].strip(';').strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fn = None
elif retry:
fn, msg, retry, wait, data = _analyse(fn, url)
if not fn:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
bad_fetch(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(url)
elif '&nzbname=' in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find('&nzbname=') + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if gzipped:
filename += '.gz'
if not data:
try:
data = fn.read()
except (IncompleteRead, IOError):
bad_fetch(future_nzo, url, T('Server could not complete request'))
fn.close()
continue
fn.close()
if '<nzb' in data and misc.get_ext(filename) != '.nzb':
filename += '.nzb'
# Sanitize filename first (also removing forbidden Windows-names)
filename = misc.sanitize_filename(filename)
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)
path = os.path.join(path, filename)
f = open(path, 'wb')
f.write(data)
f.close()
del data
# Check if nzb file
if misc.get_ext(filename) in ('.nzb', '.gz', 'bz2'):
res = dirscanner.ProcessSingleFile(filename, path, pp=pp, script=script, cat=cat, priority=priority,
nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)[0]
if res:
if res == -2:
logging.info('Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False)
continue
else:
logging.info('Unknown error fetching NZB, retry after 2 min %s', url)
when = 120
self.add(url, future_nzo, when)
# Check if a supported archive
else:
status, zf, exp_ext = dirscanner.is_archive(path)
if status == 0:
if misc.get_ext(filename) not in ('.rar', '.zip', '.7z'):
filename = filename + exp_ext
os.rename(path, path + exp_ext)
path = path + exp_ext
dirscanner.ProcessArchiveFile(filename, path, pp, script, cat, priority=priority,
nzbname=nzbname, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fn)
except:
pass
logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url)
self.add(url, future_nzo, 30)
except:
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
# Don't pound the website!
time.sleep(5.0)
(url, future_nzo, retry_count) = self.queue.get()
if not url:
continue
url = url.replace(' ', '')
try:
del_bookmark = not future_nzo
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing',
url)
continue
# Add nzbmatrix credentials if needed
url, matrix_id = _matrix_url(url)
# When still waiting for nzbmatrix wait period, requeue
if matrix_id and self.matrix_wait > time.time():
self.queue.put((url, future_nzo, retry_count))
continue
# _grab_url cannot reside in a function, because the tempfile
# would not survive the end of the function
if del_bookmark:
logging.info('Removing nzbmatrix bookmark %s', matrix_id)
else:
logging.info('Grabbing URL %s', url)
opener = urllib.FancyURLopener({})
opener.prompt_user_passwd = None
opener.addheaders = []
opener.addheader('User-Agent',
'SABnzbd+/%s' % sabnzbd.version.__version__)
opener.addheader('Accept-encoding', 'gzip')
filename = None
category = None
length = 0
nzo_info = {}
try:
fn, header = opener.retrieve(url)
except:
fn = None
if fn:
for tup in header.items():
try:
item = tup[0].lower()
value = tup[1].strip()
except:
continue
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo', ):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name', ):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item in ('content-length', ):
length = misc.int_conv(value)
if not filename:
for item in tup:
if "filename=" in item:
filename = item[item.index("filename=") +
9:].strip(';').strip('"')
if matrix_id:
fn, msg, retry, wait = _analyse_matrix(fn, matrix_id)
if retry and wait > 0:
self.matrix_wait = time.time() + wait
logging.debug('Retry URL %s after waiting', url)
self.queue.put((url, future_nzo, retry_count))
continue
category = _MATRIX_MAP.get(category, category)
else:
msg = ''
retry = True
# Check if the filepath is specified, if not, check if a retry is allowed.
if not fn:
retry_count -= 1
if retry_count > 0 and retry:
logging.info('Retry URL %s', url)
self.queue.put((url, future_nzo, retry_count))
elif not del_bookmark:
misc.bad_fetch(future_nzo, url, msg, retry=True)
continue
if del_bookmark:
continue
if not filename:
filename = os.path.basename(url) + '.nzb'
# Sanitize and trim name, preserving the extension
filename, ext = os.path.splitext(filename)
filename = misc.sanitize_foldername(filename)
filename += '.' + misc.sanitize_foldername(ext)
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# Check if nzb file
if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz'):
res = dirscanner.ProcessSingleFile(filename, fn, pp=pp, script=script, cat=cat, priority=priority, \
nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url)
if res == 0:
NzbQueue.do.remove(future_nzo.nzo_id,
add_to_history=False)
elif res == -2:
retry_count -= 1
if retry_count > 0:
logging.info('Incomplete NZB, retry %s', url)
self.queue.put((url, future_nzo, retry_count))
else:
misc.bad_fetch(future_nzo,
url,
retry=True,
content=True)
else:
misc.bad_fetch(future_nzo,
url,
retry=True,
content=True)
# Check if a supported archive
else:
if dirscanner.ProcessArchiveFile(filename,
fn,
pp,
script,
cat,
priority=priority,
url=future_nzo.url) == 0:
NzbQueue.do.remove(future_nzo.nzo_id,
add_to_history=False)
else:
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fn)
except:
pass
misc.bad_fetch(future_nzo,
url,
retry=True,
content=True)
except:
logging.error('URLGRABBER CRASHED', exc_info=True)
logging.debug("URLGRABBER Traceback: ", exc_info=True)
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
# Don't pound the website!
time.sleep(5.0)
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo and future_nzo.wait and future_nzo.wait > time.time(
):
# Requeue when too early and still active
self.add(url, future_nzo)
continue
url = url.replace(' ', '')
try:
del_bookmark = not future_nzo
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing',
url)
continue
# Add nzbmatrix credentials if needed
url, matrix_id = _matrix_url(url)
# _grab_url cannot reside in a function, because the tempfile
# would not survive the end of the function
if del_bookmark:
logging.info('Removing nzbmatrix bookmark %s', matrix_id)
else:
logging.info('Grabbing URL %s', url)
if '.nzbsrus.' in url:
opener = urllib.URLopener({})
else:
opener = urllib.FancyURLopener({})
opener.prompt_user_passwd = None
opener.addheaders = []
opener.addheader('User-Agent',
'SABnzbd+/%s' % sabnzbd.version.__version__)
if not [True for item in _BAD_GZ_HOSTS if item in url]:
opener.addheader('Accept-encoding', 'gzip')
filename = None
category = None
length = 0
nzo_info = {}
wait = 0
try:
fn, header = opener.retrieve(url)
except:
fn = None
if fn:
for tup in header.items():
try:
item = tup[0].lower()
value = tup[1].strip()
except:
continue
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo', ):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name', ):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item in ('content-length', ):
length = misc.int_conv(value)
elif item == 'retry-after':
# For NZBFinder
wait = misc.int_conv(value)
if not filename:
for item in tup:
if "filename=" in item:
filename = item[item.index("filename=") +
9:].strip(';').strip('"')
if matrix_id:
fn, msg, retry, wait = _analyse_matrix(fn, matrix_id)
if not fn:
if retry:
logging.info(msg)
logging.debug(
'Retry nzbmatrix item %s after waiting %s sec',
matrix_id, wait)
self.add(url, future_nzo, wait)
else:
logging.error(msg)
misc.bad_fetch(future_nzo,
clean_matrix_url(url),
msg,
retry=True)
continue
category = get_matrix_category(url, category)
if del_bookmark:
# No retries of nzbmatrix bookmark removals
continue
else:
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fn = None
else:
fn, msg, retry, wait = _analyse_others(fn, url)
if not fn:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
misc.bad_fetch(future_nzo, url, msg, retry=True)
continue
if not filename:
filename = os.path.basename(url) + '.nzb'
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# Check if nzb file
if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz'):
res, nzo_ids = dirscanner.ProcessSingleFile(filename, fn, pp=pp, script=script, cat=cat, priority=priority, \
nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url)
if res == 0:
NzbQueue.do.remove(future_nzo.nzo_id,
add_to_history=False)
else:
if res == -2:
logging.info(
'Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id,
add_to_history=False)
continue
else:
logging.info(
'Unknown error fetching NZB, retry after 2 min %s',
url)
when = 120
self.add(url, future_nzo, when)
# Check if a supported archive
else:
if dirscanner.ProcessArchiveFile(
filename,
fn,
pp,
script,
cat,
priority=priority,
nzbname=nzbname,
url=future_nzo.url)[0] == 0:
NzbQueue.do.remove(future_nzo.nzo_id,
add_to_history=False)
else:
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fn)
except:
pass
logging.info(
'Unknown filetype when fetching NZB, retry after 30s %s',
url)
self.add(url, future_nzo, 30)
except:
logging.error('URLGRABBER CRASHED', exc_info=True)
logging.debug("URLGRABBER Traceback: ", exc_info=True)
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
# Don't pound the website!
time.sleep(5.0)
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo and future_nzo.wait and future_nzo.wait > time.time(
):
# Re-queue when too early and still active
self.add(url, future_nzo)
continue
url = url.replace(' ', '')
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing',
url)
continue
logging.info('Grabbing URL %s', url)
req = urllib2.Request(url)
req.add_header('User-Agent',
'SABnzbd+/%s' % sabnzbd.version.__version__)
if not [True for item in _BAD_GZ_HOSTS if item in url]:
req.add_header('Accept-encoding', 'gzip')
filename = None
category = None
gzipped = False
nzo_info = {}
wait = 0
retry = True
fn = None
try:
fn = urllib2.urlopen(req)
except:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s',
error1, url)
if 'certificate_verify_failed' in error1 or 'certificateerror' in error0:
msg = T('Server %s uses an untrusted HTTPS certificate'
) % ''
retry = False
elif 'nodename nor servname provided' in error1:
msg = T('Server name does not resolve')
retry = False
elif '401' in error1 or 'unauthorized' in error1:
msg = T('Unauthorized access')
retry = False
elif '404' in error1:
msg = T('File not on server')
retry = False
new_url = dereferring(url, fn)
if new_url:
self.add(new_url, future_nzo)
continue
if fn:
for hdr in fn.headers:
try:
item = hdr.lower()
value = fn.headers[hdr]
except:
continue
if item in ('content-encoding', ) and value == 'gzip':
gzipped = True
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo', ):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name', ):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item == 'x-dnzb-password':
nzo_info['password'] = value
elif item == 'retry-after':
# For NZBFinder
wait = misc.int_conv(value)
if not filename and "filename=" in value:
filename = value[value.index("filename=") +
9:].strip(';').strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fn = None
elif retry:
fn, msg, retry, wait, data = _analyse(fn, url)
if not fn:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
bad_fetch(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(url) + '.nzb'
elif '&nzbname=' in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find('&nzbname=') + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if gzipped:
filename = filename + '.gz'
if not data:
data = fn.read()
fn.close()
# Sanatize filename first
filename = misc.sanitize_filename(filename)
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)
path = os.path.join(path, filename)
f = open(path, 'wb')
f.write(data)
f.close()
del data
# Check if nzb file
if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz',
'bz2'):
res = dirscanner.ProcessSingleFile(
filename,
path,
pp=pp,
script=script,
cat=cat,
priority=priority,
nzbname=nzbname,
nzo_info=nzo_info,
url=future_nzo.url,
keep=False,
nzo_id=future_nzo.nzo_id)[0]
if res:
if res == -2:
logging.info(
'Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id,
add_to_history=False)
continue
else:
logging.info(
'Unknown error fetching NZB, retry after 2 min %s',
url)
when = 120
self.add(url, future_nzo, when)
# Check if a supported archive
else:
if dirscanner.ProcessArchiveFile(
filename,
path,
pp,
script,
cat,
priority=priority,
nzbname=nzbname,
url=future_nzo.url,
keep=False,
nzo_id=future_nzo.nzo_id)[0]:
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fn)
except:
pass
logging.info(
'Unknown filetype when fetching NZB, retry after 30s %s',
url)
self.add(url, future_nzo, 30)
except:
logging.error(T('URLGRABBER CRASHED'), exc_info=True)
logging.debug("URLGRABBER Traceback: ", exc_info=True)
