def add_local(f):
""" Function for easily adding nzb/zip/rar/nzb.gz to sabnzbd """
if os.path.exists(f):
fn = get_filename(f)
if fn:
if get_ext(fn) in VALID_ARCHIVES:
ProcessArchiveFile(fn, f, keep=True)
elif get_ext(fn) in VALID_NZB_FILES:
ProcessSingleFile(fn, f, keep=True)
else:
logging.error("Filename not found: %s", f)
else:
logging.error("File not found: %s", f)
def add_local(f):
""" Function for easily adding nzb/zip/rar/nzb.gz to sabnzbd """
if os.path.exists(f):
fn = get_filename(f)
if fn:
if get_ext(fn) in VALID_ARCHIVES:
ProcessArchiveFile(fn, f, keep=True)
elif get_ext(fn) in VALID_NZB_FILES:
ProcessSingleFile(fn, f, keep=True)
else:
logging.error("Filename not found: %s", f)
else:
logging.error("File not found: %s", f)
def application_openFiles_(self, nsapp, filenames):
#logging.info('[osx] file open')
#logging.info('[osx] file : %s' % (filenames))
for name in filenames :
logging.info('[osx] receiving from OSX : %s' % name)
if os.path.exists(name):
fn = get_filename(name)
#logging.info('[osx] filename : %s' % (fn))
if fn:
if get_ext(name) in ('.zip', '.rar'):
#logging.info('[osx] archive')
dirscanner.ProcessArchiveFile(fn, name, keep=True)
elif get_ext(name) in ('.nzb', '.gz'):
#logging.info('[osx] nzb')
dirscanner.ProcessSingleFile(fn, name, keep=True)
def application_openFiles_(self, nsapp, filenames):
# logging.info('[osx] file open')
# logging.info('[osx] file : %s' % (filenames))
for name in filenames:
logging.info('[osx] receiving from OSX : %s', name)
if os.path.exists(name):
fn = get_filename(name)
# logging.info('[osx] filename : %s' % (fn))
if fn:
if get_ext(name) in VALID_ARCHIVES:
# logging.info('[osx] archive')
dirscanner.ProcessArchiveFile(fn, name, keep=True)
elif get_ext(name) in ('.nzb', '.gz', '.bz2'):
# logging.info('[osx] nzb')
dirscanner.ProcessSingleFile(fn, name, keep=True)
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = _file.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, _file)
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size > 314572800 and not RE_SAMPLE.search(_file) \
and get_ext(_file) not in EXCLUDED_FILE_EXTS:
return True
return False
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = os.path.normpath(_file)
else:
filepath = os.path.normpath(os.path.join(current_path, _file))
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size >= cfg.movie_rename_limit.get_int() and not RE_SAMPLE.search(_file) \
and get_ext(_file) not in EXCLUDED_FILE_EXTS:
return True
return False
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = os.path.normpath(_file)
else:
filepath = os.path.normpath(os.path.join(current_path, _file))
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size >= cfg.movie_rename_limit.get_int() and not RE_SAMPLE.search(_file) \
and get_ext(_file) not in EXCLUDED_FILE_EXTS:
return True
return False
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = _file.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, _file)
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size > 314572800 and not RE_SAMPLE.search(_file) \
and get_ext(_file) not in EXCLUDED_FILE_EXTS:
return True
return False
def rename(self, files, current_path):
""" Rename for Series
"""
logging.debug("Renaming Series")
largest = (None, None, 0)
def to_filepath(f, current_path):
if is_full_path(f):
filepath = f.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, f)
return filepath
# Create a generator of filepaths, ignore sample files and excluded files (vobs ect)
filepaths = ((file, to_filepath(file, current_path)) for file in files if not RE_SAMPLE.search(file) \
and get_ext(file) not in EXCLUDED_FILE_EXTS)
# Find the largest existing file
for file, fp in filepaths:
# If for some reason the file no longer exists, skip
if not os.path.exists(fp):
continue
size = os.stat(fp).st_size
f_file, f_fp, f_size = largest
if size > f_size:
largest = (file, fp, size)
file, filepath, size = largest
# >20MB
if filepath and size > 20971520:
tmp, self.ext = os.path.splitext(file)
self.fname = tmp
newname = "%s%s" % (self.filename_set, self.ext)
# Replace %fn with the original filename
newname = newname.replace('%fn', tmp)
newpath = os.path.join(current_path, newname)
# Replace %ext with extension
newpath = newpath.replace('%ext', self.ext)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error("Failed to rename: %s to %s", current_path,
newpath)
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, self.ext, self.filename_set)
else:
logging.debug(
'Current path already exists, skipping rename, %s',
newpath)
else:
logging.debug('Nothing to rename, %s', files)
def rename(self, files, current_path):
""" Rename for Series
"""
logging.debug("Renaming Series")
largest = (None, None, 0)
def to_filepath(f, current_path):
if is_full_path(f):
filepath = f.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, f)
return filepath
# Create a generator of filepaths, ignore sample files and excluded files (vobs ect)
filepaths = ((file, to_filepath(file, current_path)) for file in files if not RE_SAMPLE.search(file) \
and get_ext(file) not in EXCLUDED_FILE_EXTS)
# Find the largest existing file
for file, fp in filepaths:
# If for some reason the file no longer exists, skip
if not os.path.exists(fp):
continue
size = os.stat(fp).st_size
f_file, f_fp, f_size = largest
if size > f_size:
largest = (file, fp, size)
file, filepath, size = largest
# >20MB
if filepath and size > 20971520:
tmp, self.ext = os.path.splitext(file)
self.fname = tmp
newname = "%s%s" % (self.filename_set, self.ext)
# Replace %fn with the original filename
newname = newname.replace('%fn', tmp)
newpath = os.path.join(current_path, newname)
# Replace %ext with extension
newpath = newpath.replace('%ext', self.ext)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error("Failed to rename: %s to %s", current_path, newpath)
logging.info("Traceback: ", exc_info = True)
rename_similar(current_path, self.ext, self.filename_set)
else:
logging.debug('Current path already exists, skipping rename, %s', newpath)
else:
logging.debug('Nothing to rename, %s', files)
def check_encrypted_and_unwanted_files(nzo, filepath):
""" Combines check for unwanted and encrypted files to save on CPU and IO """
encrypted = False
unwanted = None
if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (
nzo.encrypted == 0 and cfg.pause_on_pwrar()):
# These checks should not break the assembler
try:
# Rarfile freezes on Windows special names, so don't try those!
if sabnzbd.WIN32 and has_win_device(filepath):
return encrypted, unwanted
# Is it even a rarfile?
if rarfile.is_rarfile(filepath):
# Open the rar
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
zf = rarfile.RarFile(filepath, all_names=True)
# Check for encryption
if nzo.encrypted == 0 and cfg.pause_on_pwrar() and (
zf.needs_password()
or is_cloaked(nzo, filepath, zf.namelist())):
# Load all passwords
passwords = get_all_passwords(nzo)
# Cloaked job?
if is_cloaked(nzo, filepath, zf.namelist()):
encrypted = True
elif not sabnzbd.HAVE_CRYPTOGRAPHY and not passwords:
# if no cryptography installed, only error when no password was set
logging.info(T('%s missing'), 'Python Cryptography')
nzo.encrypted = 1
encrypted = True
elif sabnzbd.HAVE_CRYPTOGRAPHY:
# Lets test if any of the password work
password_hit = False
for password in passwords:
if password:
logging.info(
'Trying password "%s" on job "%s"',
password, nzo.final_name)
try:
zf.setpassword(password)
except:
# On weird passwords the setpassword() will fail
# but the actual rartest() will work
pass
try:
zf.testrar()
password_hit = password
break
except rarfile.RarCRCError:
# On CRC error we can continue!
password_hit = password
break
except Exception as e:
# Did we start from the right volume?
if 'need to start extraction from a previous volume' in e[
0]:
return encrypted, unwanted
# This one failed
pass
# Did any work?
if password_hit:
# We always trust the user's input
if not nzo.password:
nzo.password = password_hit
# Don't check other files
logging.info('Password "%s" matches for job "%s"',
password_hit, nzo.final_name)
nzo.encrypted = -1
encrypted = False
else:
# Encrypted and none of them worked
nzo.encrypted = 1
encrypted = True
else:
# Don't check other files
nzo.encrypted = -1
encrypted = False
# Check for unwanted extensions
if cfg.unwanted_extensions(
) and cfg.action_on_unwanted_extensions():
for somefile in zf.namelist():
logging.debug('File contains: %s', somefile)
if get_ext(somefile).replace(
'.', '').lower() in cfg.unwanted_extensions():
logging.debug('Unwanted file %s', somefile)
unwanted = somefile
zf.close()
del zf
except:
logging.info('Error during inspection of RAR-file %s', filepath)
logging.debug('Traceback: ', exc_info=True)
return encrypted, unwanted
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo:
# Re-queue when too early and still active
if future_nzo.wait and future_nzo.wait > time.time():
self.add(url, future_nzo)
time.sleep(1.0)
continue
# Paused
if future_nzo.status == Status.PAUSED:
self.add(url, future_nzo)
time.sleep(1.0)
continue
url = url.replace(' ', '')
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing', url)
continue
filename = None
category = None
gzipped = False
nzo_info = {}
wait = 0
retry = True
fn = None
logging.info('Grabbing URL %s', url)
try:
fn = _build_request(url)
except Exception, e:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s', error1, url)
if 'certificate_verify_failed' in error1 or 'certificateerror' in error0:
msg = T('Server %s uses an untrusted HTTPS certificate') % ''
msg += ' - https://sabnzbd.org/certificate-errors'
retry = False
elif 'nodename nor servname provided' in error1:
msg = T('Server name does not resolve')
retry = False
elif '401' in error1 or 'unauthorized' in error1:
msg = T('Unauthorized access')
retry = False
elif '404' in error1:
msg = T('File not on server')
retry = False
elif hasattr(e, 'headers') and 'retry-after' in e.headers:
# Catch if the server send retry (e.headers is case-INsensitive)
wait = misc.int_conv(e.headers['retry-after'])
new_url = dereferring(url, fn)
if new_url:
self.add(new_url, future_nzo)
continue
if fn:
for hdr in fn.headers:
try:
item = hdr.lower()
value = fn.headers[hdr]
except:
continue
if item in ('content-encoding',) and value == 'gzip':
gzipped = True
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo',):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name',):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item == 'x-dnzb-password':
nzo_info['password'] = value
elif item == 'retry-after':
wait = misc.int_conv(value)
# Rating fields
if item in _RARTING_FIELDS:
nzo_info[item] = value
if not filename and "filename=" in value:
filename = value[value.index("filename=") + 9:].strip(';').strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fn = None
elif retry:
fn, msg, retry, wait, data = _analyse(fn, url)
if not fn:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
bad_fetch(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(url)
elif '&nzbname=' in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find('&nzbname=') + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if gzipped:
filename += '.gz'
if not data:
try:
data = fn.read()
except (IncompleteRead, IOError):
bad_fetch(future_nzo, url, T('Server could not complete request'))
fn.close()
continue
fn.close()
if '<nzb' in data and misc.get_ext(filename) != '.nzb':
filename += '.nzb'
# Sanitize filename first (also removing forbidden Windows-names)
filename = misc.sanitize_filename(filename)
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)
path = os.path.join(path, filename)
f = open(path, 'wb')
f.write(data)
f.close()
del data
# Check if nzb file
if misc.get_ext(filename) in ('.nzb', '.gz', 'bz2'):
res = dirscanner.ProcessSingleFile(filename, path, pp=pp, script=script, cat=cat, priority=priority,
nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)[0]
if res:
if res == -2:
logging.info('Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False)
continue
else:
logging.info('Unknown error fetching NZB, retry after 2 min %s', url)
when = 120
self.add(url, future_nzo, when)
# Check if a supported archive
else:
status, zf, exp_ext = dirscanner.is_archive(path)
if status == 0:
if misc.get_ext(filename) not in ('.rar', '.zip', '.7z'):
filename = filename + exp_ext
os.rename(path, path + exp_ext)
path = path + exp_ext
dirscanner.ProcessArchiveFile(filename, path, pp, script, cat, priority=priority,
nzbname=nzbname, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fn)
except:
pass
logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url)
self.add(url, future_nzo, 30)
except:
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo:
# Re-queue when too early and still active
if future_nzo.url_wait and future_nzo.url_wait > time.time():
self.add(url, future_nzo)
time.sleep(1.0)
continue
# Paused
if future_nzo.status == Status.PAUSED:
self.add(url, future_nzo)
time.sleep(1.0)
continue
url = url.replace(' ', '')
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing', url)
continue
filename = None
category = None
gzipped = False
nzo_info = {}
wait = 0
retry = True
fetch_request = None
logging.info('Grabbing URL %s', url)
try:
fetch_request = _build_request(url)
except Exception, e:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s', error1, url)
if 'certificate_verify_failed' in error1 or 'certificateerror' in error0:
msg = T('Server %s uses an untrusted HTTPS certificate') % ''
msg += ' - https://sabnzbd.org/certificate-errors'
retry = False
elif 'nodename nor servname provided' in error1:
msg = T('Server name does not resolve')
retry = False
elif '401' in error1 or 'unauthorized' in error1:
msg = T('Unauthorized access')
retry = False
elif '404' in error1:
msg = T('File not on server')
retry = False
elif hasattr(e, 'headers') and 'retry-after' in e.headers:
# Catch if the server send retry (e.headers is case-INsensitive)
wait = misc.int_conv(e.headers['retry-after'])
# Check if dereference is used
new_url = dereferring(url, fetch_request)
if new_url:
self.add(new_url, future_nzo)
continue
if fetch_request:
for hdr in fetch_request.headers:
try:
item = hdr.lower()
value = fetch_request.headers[hdr]
except:
continue
if item in ('content-encoding',) and value == 'gzip':
gzipped = True
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo',):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name',):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item == 'x-dnzb-password':
nzo_info['password'] = value
elif item == 'retry-after':
wait = misc.int_conv(value)
# Rating fields
if item in _RARTING_FIELDS:
nzo_info[item] = value
# Get filename from Content-Disposition header
if not filename and "filename=" in value:
filename = value[value.index("filename=") + 9:].strip(';').strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fetch_request = None
elif retry:
fetch_request, msg, retry, wait, data = _analyse(fetch_request, future_nzo)
if not fetch_request:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
self.fail_to_history(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(urllib2.unquote(url))
# URL was redirected, maybe the redirect has better filename?
# Check if the original URL has extension
if url != fetch_request.url and misc.get_ext(filename) not in VALID_NZB_FILES:
filename = os.path.basename(urllib2.unquote(fetch_request.url))
elif '&nzbname=' in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find('&nzbname=') + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if gzipped:
filename += '.gz'
if not data:
try:
data = fetch_request.read()
except (IncompleteRead, IOError):
self.fail_to_history(future_nzo, url, T('Server could not complete request'))
fetch_request.close()
continue
fetch_request.close()
if '<nzb' in data and misc.get_ext(filename) != '.nzb':
filename += '.nzb'
# Sanitize filename first (also removing forbidden Windows-names)
filename = misc.sanitize_filename(filename)
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)
path = os.path.join(path, filename)
f = open(path, 'wb')
f.write(data)
f.close()
del data
# Check if nzb file
if misc.get_ext(filename) in VALID_NZB_FILES:
res = dirscanner.ProcessSingleFile(filename, path, pp=pp, script=script, cat=cat, priority=priority,
nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)[0]
if res:
if res == -2:
logging.info('Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False)
continue
else:
logging.info('Unknown error fetching NZB, retry after 2 min %s', url)
when = 120
self.add(url, future_nzo, when)
else:
# Check if a supported archive
status, zf, exp_ext = dirscanner.is_archive(path)
if status == 0:
if misc.get_ext(filename) not in ('.rar', '.zip', '.7z'):
filename = filename + exp_ext
os.rename(path, path + exp_ext)
path = path + exp_ext
dirscanner.ProcessArchiveFile(filename, path, pp, script, cat, priority=priority,
nzbname=nzbname, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)
else:
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fetch_request)
except:
pass
logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url)
self.add(url, future_nzo, 30)
except:
def run(self):
logging.info('URLGrabber starting up')
self.shutdown = False
while not self.shutdown:
# Don't pound the website!
time.sleep(5.0)
(url, future_nzo) = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo and future_nzo.wait and future_nzo.wait > time.time():
# Re-queue when too early and still active
self.add(url, future_nzo)
continue
url = url.replace(' ', '')
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug('Dropping URL %s, job entry missing', url)
continue
logging.info('Grabbing URL %s', url)
req = urllib2.Request(url)
req.add_header('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__)
if not any(item in url for item in _BAD_GZ_HOSTS):
req.add_header('Accept-encoding', 'gzip')
filename = None
category = None
gzipped = False
nzo_info = {}
wait = 0
retry = True
fn = None
try:
fn = urllib2.urlopen(req)
except:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s', error1, url)
if 'certificate_verify_failed' in error1 or 'certificateerror' in error0:
msg = T('Server %s uses an untrusted HTTPS certificate') % ''
retry = False
elif 'nodename nor servname provided' in error1:
msg = T('Server name does not resolve')
retry = False
elif '401' in error1 or 'unauthorized' in error1:
msg = T('Unauthorized access')
retry = False
elif '404' in error1:
msg = T('File not on server')
retry = False
new_url = dereferring(url, fn)
if new_url:
self.add(new_url, future_nzo)
continue
if fn:
for hdr in fn.headers:
try:
item = hdr.lower()
value = fn.headers[hdr]
except:
continue
if item in ('content-encoding',) and value == 'gzip':
gzipped = True
if item in ('category_id', 'x-dnzb-category'):
category = value
elif item in ('x-dnzb-moreinfo',):
nzo_info['more_info'] = value
elif item in ('x-dnzb-name',):
filename = value
if not filename.endswith('.nzb'):
filename += '.nzb'
elif item == 'x-dnzb-propername':
nzo_info['propername'] = value
elif item == 'x-dnzb-episodename':
nzo_info['episodename'] = value
elif item == 'x-dnzb-year':
nzo_info['year'] = value
elif item == 'x-dnzb-failure':
nzo_info['failure'] = value
elif item == 'x-dnzb-details':
nzo_info['details'] = value
elif item == 'x-dnzb-password':
nzo_info['password'] = value
elif item == 'retry-after':
# For NZBFinder
wait = misc.int_conv(value)
# Rating fields
if item in _RARTING_FIELDS:
nzo_info[item] = value
if not filename and "filename=" in value:
filename = value[value.index("filename=") + 9:].strip(';').strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ''
retry = True
fn = None
elif retry:
fn, msg, retry, wait, data = _analyse(fn, url)
if not fn:
if retry:
logging.info('Retry URL %s', url)
self.add(url, future_nzo, wait)
else:
bad_fetch(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(url)
elif '&nzbname=' in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find('&nzbname=') + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == '*') and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if gzipped:
filename += '.gz'
if not data:
data = fn.read()
fn.close()
if '<nzb' in data and misc.get_ext(filename) != '.nzb':
filename += '.nzb'
# Sanatize filename first
filename = misc.sanitize_filename(filename)
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)
path = os.path.join(path, filename)
f = open(path, 'wb')
f.write(data)
f.close()
del data
# Check if nzb file
if misc.get_ext(filename) in ('.nzb', '.gz', 'bz2'):
res = dirscanner.ProcessSingleFile(filename, path, pp=pp, script=script, cat=cat, priority=priority,
nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)[0]
if res:
if res == -2:
logging.info('Incomplete NZB, retry after 5 min %s', url)
when = 300
elif res == -1:
# Error, but no reason to retry. Warning is already given
NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False)
continue
else:
logging.info('Unknown error fetching NZB, retry after 2 min %s', url)
when = 120
self.add(url, future_nzo, when)
# Check if a supported archive
else:
status, zf, exp_ext = dirscanner.is_archive(path)
if status == 0:
if misc.get_ext(filename) not in ('.rar', '.zip', '.7z'):
filename = filename + exp_ext
os.rename(path, path + exp_ext)
path = path + exp_ext
dirscanner.ProcessArchiveFile(filename, path, pp, script, cat, priority=priority,
nzbname=nzbname, url=future_nzo.url, keep=False,
nzo_id=future_nzo.nzo_id)
# Not a supported filetype, not an nzb (text/html ect)
try:
os.remove(fn)
except:
pass
logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url)
self.add(url, future_nzo, 30)
except:
logging.error(T('URLGRABBER CRASHED'), exc_info=True)
logging.debug("URLGRABBER Traceback: ", exc_info=True)
def check_encrypted_and_unwanted_files(nzo, filepath):
""" Combines check for unwanted and encrypted files to save on CPU and IO """
encrypted = False
unwanted = None
if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (nzo.encrypted == 0 and cfg.pause_on_pwrar()):
# These checks should not break the assembler
try:
# Rarfile freezes on Windows special names, so don't try those!
if sabnzbd.WIN32 and has_win_device(filepath):
return encrypted, unwanted
# Is it even a rarfile?
if is_rarfile(filepath):
# Open the rar
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
zf = rarfile.RarFile(filepath, all_names=True)
# Check for encryption
if nzo.encrypted == 0 and cfg.pause_on_pwrar() and (zf.needs_password() or is_cloaked(nzo, filepath, zf.namelist())):
# Load all passwords
passwords = get_all_passwords(nzo)
# Cloaked job?
if is_cloaked(nzo, filepath, zf.namelist()):
encrypted = True
elif not sabnzbd.HAVE_CRYPTOGRAPHY and not passwords:
# if no cryptography installed, only error when no password was set
logging.info(T('%s missing'), 'Python Cryptography')
nzo.encrypted = 1
encrypted = True
elif sabnzbd.HAVE_CRYPTOGRAPHY:
# Lets test if any of the password work
password_hit = False
for password in passwords:
if password:
logging.info('Trying password "%s" on job "%s"', password, nzo.final_name)
try:
zf.setpassword(password)
except:
# On weird passwords the setpassword() will fail
# but the actual rartest() will work
pass
try:
zf.testrar()
password_hit = password
break
except rarfile.RarCRCError:
# On CRC error we can continue!
password_hit = password
break
except Exception as e:
# Did we start from the right volume?
if 'need to start extraction from a previous volume' in e[0]:
return encrypted, unwanted
# This one failed
pass
# Did any work?
if password_hit:
# We always trust the user's input
if not nzo.password:
nzo.password = password_hit
# Don't check other files
logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name)
nzo.encrypted = -1
encrypted = False
else:
# Encrypted and none of them worked
nzo.encrypted = 1
encrypted = True
else:
# Don't check other files
nzo.encrypted = -1
encrypted = False
# Check for unwanted extensions
if cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions():
for somefile in zf.namelist():
logging.debug('File contains: %s', somefile)
if get_ext(somefile).replace('.', '').lower() in cfg.unwanted_extensions():
logging.debug('Unwanted file %s', somefile)
unwanted = somefile
zf.close()
del zf
except:
logging.info('Error during inspection of RAR-file %s', filepath)
logging.debug('Traceback: ', exc_info=True)
return encrypted, unwanted
