def add(self, nzo, save=True, quiet=False):
if not nzo.nzo_id:
nzo.nzo_id = sabnzbd.get_new_id("nzo", nzo.workpath,
self.__nzo_table)
# If no files are to be downloaded anymore, send to postproc
if not nzo.files and not nzo.futuretype:
self.end_job(nzo)
return nzo.nzo_id
# Reset try_lists
nzo.reset_try_list()
if nzo.nzo_id:
nzo.deleted = False
priority = nzo.priority
if sabnzbd.scheduler.analyse(False, priority):
nzo.status = Status.PAUSED
self.__nzo_table[nzo.nzo_id] = nzo
if priority > HIGH_PRIORITY:
# Top and repair priority items are added to the top of the queue
self.__nzo_list.insert(0, nzo)
elif priority == LOW_PRIORITY:
self.__nzo_list.append(nzo)
else:
# for high priority we need to add the item at the bottom
# of any other high priority items above the normal priority
# for normal priority we need to add the item at the bottom
# of the normal priority items above the low priority
if self.__nzo_list:
pos = 0
added = False
for position in self.__nzo_list:
if position.priority < priority:
self.__nzo_list.insert(pos, nzo)
added = True
break
pos += 1
if not added:
# if there are no other items classed as a lower priority
# then it will be added to the bottom of the queue
self.__nzo_list.append(nzo)
else:
# if the queue is empty then simple append the item to the bottom
self.__nzo_list.append(nzo)
if save:
self.save(nzo)
if not (quiet or nzo.status == Status.FETCHING):
notifier.send_notification(T("NZB added to queue"),
nzo.filename, "download", nzo.cat)
if not quiet and cfg.auto_sort():
try:
field, direction = cfg.auto_sort().split()
self.sort_queue(field, direction)
except ValueError:
pass
return nzo.nzo_id
def add(self, nzo, save=True, quiet=False):
if 0:
assert isinstance(nzo, NzbObject) # Assert only for debug purposes
if not nzo.nzo_id:
nzo.nzo_id = sabnzbd.get_new_id('nzo', nzo.workpath,
self.__nzo_table)
# If no files are to be downloaded anymore, send to postproc
if not nzo.files and not nzo.futuretype:
self.end_job(nzo)
return ''
# Reset try_lists
nzo.reset_try_list()
self.reset_try_list()
if nzo.nzo_id:
nzo.deleted = False
priority = nzo.priority
if sabnzbd.scheduler.analyse(False, priority):
nzo.status = Status.PAUSED
self.__nzo_table[nzo.nzo_id] = nzo
if priority > HIGH_PRIORITY:
# Top and repair priority items are added to the top of the queue
self.__nzo_list.insert(0, nzo)
elif priority == LOW_PRIORITY:
self.__nzo_list.append(nzo)
else:
# for high priority we need to add the item at the bottom
# of any other high priority items above the normal priority
# for normal priority we need to add the item at the bottom
# of the normal priority items above the low priority
if self.__nzo_list:
pos = 0
added = False
for position in self.__nzo_list:
if position.priority < priority:
self.__nzo_list.insert(pos, nzo)
added = True
break
pos += 1
if not added:
# if there are no other items classed as a lower priority
# then it will be added to the bottom of the queue
self.__nzo_list.append(nzo)
else:
# if the queue is empty then simple append the item to the bottom
self.__nzo_list.append(nzo)
if save:
self.save(nzo)
if not (quiet or nzo.status in ('Fetching', )):
notifier.send_notification(T('NZB added to queue'),
nzo.filename, 'download')
if cfg.auto_sort():
self.sort_by_avg_age()
return nzo.nzo_id
def add(self, nzo, save=True, quiet=False):
if 0: assert isinstance(nzo, NzbObject) # Assert only for debug purposes
if not nzo.nzo_id:
nzo.nzo_id = sabnzbd.get_new_id('nzo', nzo.workpath, self.__nzo_table)
# If no files are to be downloaded anymore, send to postproc
if not nzo.files and not nzo.futuretype:
self.end_job(nzo)
return ''
# Reset try_lists
nzo.reset_try_list()
self.reset_try_list()
if nzo.nzo_id:
nzo.deleted = False
priority = nzo.priority
if sabnzbd.scheduler.analyse(False, priority):
nzo.status = Status.PAUSED
self.__nzo_table[nzo.nzo_id] = nzo
if priority > HIGH_PRIORITY:
# Top and repair priority items are added to the top of the queue
self.__nzo_list.insert(0, nzo)
elif priority == LOW_PRIORITY:
self.__nzo_list.append(nzo)
else:
# for high priority we need to add the item at the bottom
# of any other high priority items above the normal priority
# for normal priority we need to add the item at the bottom
# of the normal priority items above the low priority
if self.__nzo_list:
pos = 0
added = False
for position in self.__nzo_list:
if position.priority < priority:
self.__nzo_list.insert(pos, nzo)
added = True
break
pos += 1
if not added:
# if there are no other items classed as a lower priority
# then it will be added to the bottom of the queue
self.__nzo_list.append(nzo)
else:
# if the queue is empty then simple append the item to the bottom
self.__nzo_list.append(nzo)
if save:
self.save(nzo)
if not (quiet or nzo.status in ('Fetching',)):
notifier.send_notification(T('NZB added to queue'), nzo.filename, 'download')
if cfg.auto_sort():
self.sort_by_avg_age()
return nzo.nzo_id
def add(self, nzo, save=True, quiet=False):
assert isinstance(nzo, NzbObject)
if not nzo.nzo_id:
nzo.nzo_id = sabnzbd.get_new_id("nzo", nzo.workpath, self.__nzo_table)
# If no files are to be downloaded anymore, send to postproc
if not nzo.files and not nzo.futuretype:
sabnzbd.remove_data(nzo.nzo_id, nzo.workpath)
sabnzbd.proxy_postproc(nzo)
return ""
# Reset try_lists
nzo.reset_try_list()
self.reset_try_list()
if nzo.nzo_id:
nzo.deleted = False
priority = nzo.priority
self.__nzo_table[nzo.nzo_id] = nzo
if priority > HIGH_PRIORITY:
# Top and repair priority items are added to the top of the queue
self.__nzo_list.insert(0, nzo)
elif priority == LOW_PRIORITY:
self.__nzo_list.append(nzo)
else:
# for high priority we need to add the item at the bottom
# of any other high priority items above the normal priority
# for normal priority we need to add the item at the bottom
# of the normal priority items above the low priority
if self.__nzo_list:
pos = 0
added = False
for position in self.__nzo_list:
if position.priority < priority:
self.__nzo_list.insert(pos, nzo)
added = True
break
pos += 1
if not added:
# if there are no other items classed as a lower priority
# then it will be added to the bottom of the queue
self.__nzo_list.append(nzo)
else:
# if the queue is empty then simple append the item to the bottom
self.__nzo_list.append(nzo)
if save:
self.save(nzo)
if not (quiet or nzo.status in ("Fetching",)):
growler.send_notification(T("NZB added to queue"), nzo.filename, "download")
if cfg.auto_sort():
self.sort_by_avg_age()
return nzo.nzo_id
def add(self, nzo, save=True, quiet=False):
assert isinstance(nzo, NzbObject)
if not nzo.nzo_id:
nzo.nzo_id = sabnzbd.get_new_id('nzo', nzo.workpath, self.__nzo_table)
# If no files are to be downloaded anymore, send to postproc
if not nzo.files and not nzo.futuretype:
sabnzbd.remove_data(nzo.nzo_id, nzo.workpath)
sabnzbd.proxy_postproc(nzo)
return
# Reset try_lists
nzo.reset_try_list()
self.reset_try_list()
if nzo.nzo_id:
nzo.deleted = False
priority = nzo.priority
self.__nzo_table[nzo.nzo_id] = nzo
if priority > HIGH_PRIORITY:
#Top and repair priority items are added to the top of the queue
self.__nzo_list.insert(0, nzo)
elif priority == LOW_PRIORITY:
self.__nzo_list.append(nzo)
else:
#for high priority we need to add the item at the bottom
#of any other high priority items above the normal priority
#for normal priority we need to add the item at the bottom
#of the normal priority items above the low priority
if self.__nzo_list:
pos = 0
added = False
for position in self.__nzo_list:
if position.priority < priority:
self.__nzo_list.insert(pos, nzo)
added = True
break
pos += 1
if not added:
#if there are no other items classed as a lower priority
#then it will be added to the bottom of the queue
self.__nzo_list.append(nzo)
else:
#if the queue is empty then simple append the item to the bottom
self.__nzo_list.append(nzo)
if save:
self.save(nzo)
if not (quiet or nzo.status in ('Fetching',)):
osx.sendGrowlMsg(T('NZB added to queue'), nzo.filename, osx.NOTIFICATION['download'])
if cfg.auto_sort():
self.sort_by_avg_age()
def run(self):
self.shutdown = False
while not self.shutdown:
# Set NzbObject object to None so reference from this thread
# does not keep the object alive in the future (see #1628)
future_nzo = None
url, future_nzo = self.queue.get()
if not url:
# stop signal, go test self.shutdown
continue
if future_nzo:
# Re-queue when too early and still active
if future_nzo.url_wait and future_nzo.url_wait > time.time():
self.add(url, future_nzo)
time.sleep(1.0)
continue
# Paused
if future_nzo.status == Status.PAUSED:
self.add(url, future_nzo)
time.sleep(1.0)
continue
url = url.replace(" ", "")
try:
if future_nzo:
# If nzo entry deleted, give up
try:
deleted = future_nzo.deleted
except AttributeError:
deleted = True
if deleted:
logging.debug("Dropping URL %s, job entry missing",
url)
continue
filename = None
category = None
nzo_info = {}
wait = 0
retry = True
fetch_request = None
logging.info("Grabbing URL %s", url)
try:
fetch_request = _build_request(url)
except Exception as e:
# Cannot list exceptions here, because of unpredictability over platforms
error0 = str(sys.exc_info()[0]).lower()
error1 = str(sys.exc_info()[1]).lower()
logging.debug('Error "%s" trying to get the url %s',
error1, url)
if "certificate_verify_failed" in error1 or "certificateerror" in error0:
msg = T("Server %s uses an untrusted HTTPS certificate"
) % ""
msg += " - https://sabnzbd.org/certificate-errors"
retry = False
elif "nodename nor servname provided" in error1:
msg = T("Server name does not resolve")
retry = False
elif "401" in error1 or "unauthorized" in error1:
msg = T("Unauthorized access")
retry = False
elif "404" in error1:
msg = T("File not on server")
retry = False
elif hasattr(e, "headers") and "retry-after" in e.headers:
# Catch if the server send retry (e.headers is case-INsensitive)
wait = misc.int_conv(e.headers["retry-after"])
if fetch_request:
for hdr in fetch_request.headers:
try:
item = hdr.lower()
value = fetch_request.headers[hdr]
except:
continue
if item in ("category_id", "x-dnzb-category"):
category = value
elif item in ("x-dnzb-moreinfo", ):
nzo_info["more_info"] = value
elif item in ("x-dnzb-name", ):
filename = value
if not filename.endswith(".nzb"):
filename += ".nzb"
elif item == "x-dnzb-propername":
nzo_info["propername"] = value
elif item == "x-dnzb-episodename":
nzo_info["episodename"] = value
elif item == "x-dnzb-year":
nzo_info["year"] = value
elif item == "x-dnzb-failure":
nzo_info["failure"] = value
elif item == "x-dnzb-details":
nzo_info["details"] = value
elif item == "x-dnzb-password":
nzo_info["password"] = value
elif item == "retry-after":
wait = misc.int_conv(value)
# Rating fields
if item in _RARTING_FIELDS:
nzo_info[item] = value
# Get filename from Content-Disposition header
if not filename and "filename=" in value:
filename = value[value.index("filename=") +
9:].strip(";").strip('"')
if wait:
# For sites that have a rate-limiting attribute
msg = ""
retry = True
fetch_request = None
elif retry:
fetch_request, msg, retry, wait, data = _analyse(
fetch_request, future_nzo)
if not fetch_request:
if retry:
logging.info("Retry URL %s", url)
self.add(url, future_nzo, wait)
else:
self.fail_to_history(future_nzo, url, msg)
continue
if not filename:
filename = os.path.basename(urllib.parse.unquote(url))
# URL was redirected, maybe the redirect has better filename?
# Check if the original URL has extension
if (url != fetch_request.geturl()
and sabnzbd.filesystem.get_ext(filename)
not in VALID_NZB_FILES + VALID_ARCHIVES):
filename = os.path.basename(
urllib.parse.unquote(fetch_request.geturl()))
elif "&nzbname=" in filename:
# Sometimes the filename contains the full URL, duh!
filename = filename[filename.find("&nzbname=") + 9:]
pp = future_nzo.pp
script = future_nzo.script
cat = future_nzo.cat
if (cat is None or cat == "*") and category:
cat = misc.cat_convert(category)
priority = future_nzo.priority
nzbname = future_nzo.custom_name
# process data
if not data:
try:
data = fetch_request.read()
except (IncompleteRead, IOError):
self.fail_to_history(
future_nzo, url,
T("Server could not complete request"))
fetch_request.close()
continue
fetch_request.close()
if b"<nzb" in data and sabnzbd.filesystem.get_ext(
filename) != ".nzb":
filename += ".nzb"
# Sanitize filename first (also removing forbidden Windows-names)
filename = sabnzbd.filesystem.sanitize_filename(filename)
# If no filename, make one
if not filename:
filename = sabnzbd.get_new_id(
"url",
os.path.join(cfg.admin_dir.get_path(),
FUTURE_Q_FOLDER))
# Write data to temp file
path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER,
filename)
with open(path, "wb") as temp_nzb:
temp_nzb.write(data)
# Check if nzb file
if sabnzbd.filesystem.get_ext(
filename) in VALID_ARCHIVES + VALID_NZB_FILES:
res, _ = sabnzbd.add_nzbfile(
path,
pp=pp,
script=script,
cat=cat,
priority=priority,
nzbname=nzbname,
nzo_info=nzo_info,
url=future_nzo.url,
keep=False,
password=future_nzo.password,
nzo_id=future_nzo.nzo_id,
)
# -2==Error/retry, -1==Error, 0==OK, 1==Empty
if res == -2:
logging.info("Incomplete NZB, retry after 5 min %s",
url)
self.add(url, future_nzo, when=300)
elif res == -1:
# Error already thrown
self.fail_to_history(future_nzo, url)
elif res == 1:
# No NZB-files inside archive
self.fail_to_history(future_nzo, url,
T("Empty NZB file %s") % filename)
else:
logging.info(
"Unknown filetype when fetching NZB, retry after 30s %s",
url)
self.add(url, future_nzo, 30)
# Always clean up what we wrote to disk
try:
sabnzbd.filesystem.remove_file(path)
except:
pass
except:
logging.error(T("URLGRABBER CRASHED"), exc_info=True)
logging.debug("URLGRABBER Traceback: ", exc_info=True)
