def change_cat(self, nzo_ids, cat): for nzo_id in [item.strip() for item in nzo_ids.split(',')]: if nzo_id in self.__nzo_table: nzo = self.__nzo_table[nzo_id] nzo.cat, pp, nzo.script, prio = cat_to_opts(cat) nzo.set_pp(pp) self.set_priority(nzo_id, prio)
def fail_to_history(self, nzo, url, msg='', content=False): """ Create History entry for failed URL Fetch msg: message to be logged content: report in history that cause is a bad NZB file """ # Remove the "Trying to fetch" part if url: nzo.filename = url nzo.final_name = url.strip() if content: # Bad content msg = T('Unusable NZB file') else: # Failed fetch msg = T('URL Fetching failed; %s') % unicoder(msg) # Mark as failed nzo.status = Status.FAILED nzo.fail_msg = msg notifier.send_notification( T('URL Fetching failed; %s') % '', '%s\n%s' % (msg, url), 'other', nzo.cat) if cfg.email_endjob() > 0: emailer.badfetch_mail(msg, url) # Parse category to make sure script is set correctly after a grab nzo.cat, _, nzo.script, _ = misc.cat_to_opts(nzo.cat, script=nzo.script) # Add to history and run script if desired NzbQueue.do.remove(nzo.nzo_id, add_to_history=False) PostProcessor.do.process(nzo)
def fail_to_history(nzo: NzbObject, url: str, msg="", content=False): """Create History entry for failed URL Fetch msg: message to be logged content: report in history that cause is a bad NZB file """ # Remove the "Trying to fetch" part if url: nzo.filename = url nzo.final_name = url.strip() if content: # Bad content msg = T("Unusable NZB file") else: # Failed fetch msg = T("URL Fetching failed; %s") % msg # Mark as failed nzo.set_unpack_info("Source", msg) nzo.fail_msg = msg notifier.send_notification( T("URL Fetching failed; %s") % "", "%s\n%s" % (msg, url), "failed", nzo.cat) if cfg.email_endjob() > 0: emailer.badfetch_mail(msg, url) # Parse category to make sure script is set correctly after a grab nzo.cat, _, nzo.script, _ = misc.cat_to_opts(nzo.cat, script=nzo.script) # Add to history and run script if desired sabnzbd.NzbQueue.remove(nzo.nzo_id) sabnzbd.PostProcessor.process(nzo)
def fail_to_history(self, nzo, url, msg='', content=False): """ Create History entry for failed URL Fetch msg: message to be logged content: report in history that cause is a bad NZB file """ # Remove the "Trying to fetch" part if url: nzo.filename = url nzo.final_name = url.strip() if content: # Bad content msg = T('Unusable NZB file') else: # Failed fetch msg = T('URL Fetching failed; %s') % unicoder(msg) # Mark as failed nzo.status = Status.FAILED nzo.fail_msg = msg notifier.send_notification(T('URL Fetching failed; %s') % '', '%s\n%s' % (msg, url), 'other', nzo.cat) if cfg.email_endjob() > 0: emailer.badfetch_mail(msg, url) # Parse category to make sure script is set correctly after a grab nzo.cat, _, nzo.script, _ = misc.cat_to_opts(nzo.cat, script=nzo.script) # Add to history and run script if desired NzbQueue.do.remove(nzo.nzo_id, add_to_history=False) PostProcessor.do.process(nzo)
def test_cat_to_opts(self): # Need to create the Default category, as we would in normal instance # Otherwise it will try to save the config ConfigCat("*", { "pp": 3, "script": "None", "priority": NORMAL_PRIORITY }) assert ("*", 3, "None", NORMAL_PRIORITY) == misc.cat_to_opts("*") assert ("*", 3, "None", NORMAL_PRIORITY) == misc.cat_to_opts("Nonsense") assert ("*", 1, "None", NORMAL_PRIORITY) == misc.cat_to_opts("*", pp=1) assert ("*", 1, "test.py", NORMAL_PRIORITY) == misc.cat_to_opts("*", pp=1, script="test.py") ConfigCat("movies", {"priority": HIGH_PRIORITY, "script": "test.py"}) assert ("movies", 3, "test.py", HIGH_PRIORITY) == misc.cat_to_opts("movies") assert ("movies", 1, "test.py", HIGH_PRIORITY) == misc.cat_to_opts("movies", pp=1) assert ("movies", 1, "not_test.py", HIGH_PRIORITY) == misc.cat_to_opts("movies", pp=1, script="not_test.py") assert ("movies", 3, "test.py", TOP_PRIORITY) == misc.cat_to_opts("movies", priority=TOP_PRIORITY) # If the category has DEFAULT_PRIORITY, it should use the priority of the *-category (NORMAL_PRIORITY) # If the script-name is Default for a category, it should use the script of the *-category (None) ConfigCat("software", { "priority": DEFAULT_PRIORITY, "script": "Default" }) assert ("software", 3, "None", NORMAL_PRIORITY) == misc.cat_to_opts("software") assert ("software", 3, "None", TOP_PRIORITY) == misc.cat_to_opts("software", priority=TOP_PRIORITY)
def change_cat(self, nzo_ids, cat, explicit_priority=None): result = 0 for nzo_id in [item.strip() for item in nzo_ids.split(',')]: if nzo_id in self.__nzo_table: nzo = self.__nzo_table[nzo_id] nzo.cat, pp, nzo.script, prio = cat_to_opts(cat) nzo.set_pp(pp) if explicit_priority is None: self.set_priority(nzo_id, prio) result += 1 return result
def run(self): """ Process the queue (including waits and retries) """ from sabnzbd.nzbqueue import NzbQueue def sleeper(delay): for n in range(delay): if not self.shutdown: time.sleep(1.05) self.shutdown = False msgid = None while not self.shutdown: if not msgid: (msgid, nzo) = self.queue.get() if self.shutdown or not msgid: break logging.debug("Popping msgid %s", msgid) filename, data, newzbin_cat, nzo_info = _grabnzb(msgid) if filename and data: filename = name_fixer(filename) pp = nzo.pp script = nzo.script cat = nzo.cat if cat == '*' or not cat: cat = cat_convert(newzbin_cat) priority = nzo.priority nzbname = nzo.custom_name cat, pp, script, priority = cat_to_opts(cat, pp, script, priority) try: sabnzbd.nzbqueue.insert_future_nzo(nzo, filename, msgid, data, pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname, nzo_info=nzo_info) except: logging.error(Ta('Failed to update newzbin job %s'), msgid) logging.info("Traceback: ", exc_info = True) NzbQueue.do.remove(nzo.nzo_id, False) msgid = None else: if filename: sleeper(int(filename)) else: # Fatal error, give up on this one bad_fetch(nzo, msgid, msg=nzo_info, retry=True) msgid = None osx.sendGrowlMsg(T('NZB added to queue'),filename,osx.NOTIFICATION['download']) # Keep some distance between the grabs sleeper(5) logging.debug('Stopping MSGIDGrabber')
def run(self): """ Process the queue (including waits and retries) """ from sabnzbd.nzbqueue import NzbQueue self.shutdown = False while not self.shutdown: time.sleep(5) (msgid, nzo) = self.queue.get() if self.shutdown or not msgid: break if nzo.wait and nzo.wait > time.time(): self.grab(msgid, nzo) continue logging.debug("Popping msgid %s", msgid) filename, data, newzbin_cat, nzo_info = _grabnzb(msgid) if filename and data: filename = name_fixer(filename) pp = nzo.pp script = nzo.script cat = nzo.cat if cat == '*' or not cat: cat = cat_convert(newzbin_cat) priority = nzo.priority nzbname = nzo.custom_name cat, pp, script, priority = cat_to_opts(cat, pp, script, priority) try: sabnzbd.nzbqueue.insert_future_nzo(nzo, filename, msgid, data, pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname, nzo_info=nzo_info) nzo.url = format_source_url(str(msgid)) except: logging.error(Ta('Failed to update newzbin job %s'), msgid) logging.info("Traceback: ", exc_info = True) NzbQueue.do.remove(nzo.nzo_id, False) msgid = None else: if filename: self.grab(msgid, nzo, float(filename)) else: # Fatal error, give up on this one bad_fetch(nzo, msgid, msg=nzo_info, retry=True) msgid = None if msgid: growler.send_notification(T('NZB added to queue'), filename, 'download') logging.debug('Stopping MSGIDGrabber')
def change_cat(self, nzo_ids, cat, explicit_priority=None): result = 0 for nzo_id in [item.strip() for item in nzo_ids.split(",")]: if nzo_id in self.__nzo_table: nzo = self.__nzo_table[nzo_id] nzo.cat, pp, nzo.script, prio = cat_to_opts(cat) logging.info("Set cat=%s for job %s", cat, nzo.final_name) nzo.set_pp(pp) if explicit_priority is None: self.set_priority(nzo_id, prio) # Abort any ongoing unpacking if the category changed nzo.abort_direct_unpacker() result += 1 return result
def change_cat(self, nzo_ids, cat, explicit_priority=None): result = 0 for nzo_id in [item.strip() for item in nzo_ids.split(',')]: if nzo_id in self.__nzo_table: nzo = self.__nzo_table[nzo_id] nzo.cat, pp, nzo.script, prio = cat_to_opts(cat) logging.info('Set cat=%s for job %s', cat, nzo.final_name) nzo.set_pp(pp) if explicit_priority is None: self.set_priority(nzo_id, prio) # Abort any ongoing unpacking if the category changed nzo.abort_direct_unpacker() result += 1 return result
def insert_future(self, future, filename, msgid, data, pp=None, script=None, cat=None, priority=NORMAL_PRIORITY, nzbname=None, nzo_info=None): """ Refresh a placeholder nzo with an actual nzo """ assert isinstance(future, NzbObject) if nzo_info is None: nzo_info = {} nzo_id = future.nzo_id if nzo_id in self.__nzo_table: try: sabnzbd.remove_data(nzo_id, future.workpath) logging.info("Regenerating item: %s", nzo_id) r, u, d = future.repair_opts if not r is None: pp = sabnzbd.opts_to_pp(r, u, d) scr = future.script if scr is None: scr = script categ = future.cat if categ is None: categ = cat categ, pp, script, priority = cat_to_opts(categ, pp, script, priority) # Remember old priority old_prio = future.priority try: future.__init__(filename, msgid, pp, scr, nzb=data, futuretype=False, cat=categ, priority=priority, nzbname=nzbname, nzo_info=nzo_info) future.nzo_id = nzo_id self.save(future) except ValueError: self.remove(nzo_id, False) except TypeError: self.remove(nzo_id, False) # Make sure the priority is changed now that we know the category if old_prio != priority: future.priority = None self.set_priority(future.nzo_id, priority) if cfg.auto_sort(): self.sort_by_avg_age() self.reset_try_list() except: logging.error(Ta('Error while adding %s, removing'), nzo_id) logging.info("Traceback: ", exc_info = True) self.remove(nzo_id, False) else: logging.info("Item %s no longer in queue, omitting", nzo_id)
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(T('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info=True) return T('Incorrect RSS feed description "%s"') % feed uris = feeds.uri() defCat = feeds.cat() import sabnzbd.api if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) if filter[3] in ('<', '>', 'F', 'S'): regexes.append(filter[4]) else: regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Read the RSS feed msg = None entries = None if readout: all_entries = [] for uri in uris: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) feed_parsed = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not feed_parsed: msg = T('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) status = feed_parsed.get('status', 999) if status in (401, 402, 403): msg = T('Do not have valid authentication for feed %s') % feed logging.info(msg) if 500 <= status <= 599: msg = T('Server side error (server code %s); could not get %s on %s') % (status, feed, uri) logging.info(msg) entries = feed_parsed.get('entries') if 'bozo_exception' in feed_parsed and not entries: msg = str(feed_parsed['bozo_exception']) if 'CERTIFICATE_VERIFY_FAILED' in msg: msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri) msg += ' - https://sabnzbd.org/certificate-errors' logging.error(msg) else: msg = T('Failed to retrieve RSS from %s: %s') % (uri, xml_name(msg)) logging.info(msg) if not entries: msg = T('RSS Feed %s was empty') % uri logging.info(msg) all_entries.extend(entries) entries = all_entries # In case of a new feed if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] # Error in readout or now new readout if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category, size, age, season, episode = _get_link(entry) except (AttributeError, IndexError): logging.info(T('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info=True) return T('Incompatible feed') title = entry.title # If there's multiple feeds, remove the duplicates based on title and size if len(uris) > 1: skip_job = False for job_link, job in jobs.items(): # Allow 5% size deviation because indexers might have small differences for same release if job.get('title') == title and link != job_link and (job.get('size')*0.95) < size < (job.get('size')*1.05): logging.info("Ignoring job %s from other feed", title) skip_job = True break if skip_job: continue else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None title = jobs[link].get('title', '') size = jobs[link].get('size', 0L) age = jobs[link].get('age') season = jobs[link].get('season', 0) episode = jobs[link].get('episode', 0) if link: # Make sure spaces are quoted in the URL link = link.strip().replace(' ', '%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', title) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 if ('F' in reTypes or 'S' in reTypes) and (not season or not episode): season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3] # Match against all filters until an positive or negative match logging.debug('Size %s', size) for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break elif reTypes[n] == '<' and size and from_units(regexes[n]) < size: # "Size at most" : too large logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == '>' and size and from_units(regexes[n]) > size: # "Size at least" : too small logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == 'F' and not ep_match(season, episode, regexes[n]): # "Starting from SxxEyy", too early episode logging.debug('Filter requirement match on rule %d', n) result = False break elif reTypes[n] == 'S' and season and episode and ep_match(season, episode, regexes[n], title): logging.debug('Filter matched on rule %d', n) result = True break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if not result and defCat: # Apply Feed-category on non-matched items myCat = defCat elif result and notdefault(reCats[n]): # Use the matched info myCat = reCats[n] elif category and not defCat: # No result and no Feed-category myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts(myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and self.check_duplicate(title): if cfg.no_dupes() == 1: # Dupe-detection: Discard logging.info("Ignoring duplicate job %s", title) continue elif cfg.no_dupes() == 3: # Dupe-detection: Fail # We accept it so the Queue can send it to the History logging.info("Found duplicate job %s", title) else: # Dupe-detection: Pause myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, feed, link, title, size, age, season, episode, 'G', category, myCat, myPP, myScript, act, star, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, feed, link, title, size, age, season, episode, 'B', category, myCat, myPP, myScript, False, star, priority=myPrio, rule=str(n)) # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return msg
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False def dup_title(title): """ Check if this title was in this or other feeds Return matching feed name """ title = title.lower() for fd in self.jobs: for lk in self.jobs[fd]: item = self.jobs[fd][lk] if item.get('status', ' ')[0] == 'D' and \ item.get('title', '').lower() == title: return fd return '' if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(T('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info=True) return T('Incorrect RSS feed description "%s"') % feed uri = feeds.uri() defCat = feeds.cat() import sabnzbd.api if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) if filter[3] in ('<', '>', 'F'): regexes.append(filter[4]) else: regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Check for nzbs.org if 'nzbs.org/' in uri and '&dl=1' not in uri: uri += '&dl=1' # Read the RSS feed msg = None entries = None if readout: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) d = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not d: msg = T('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) return unicoder(msg) status = d.get('status', 999) if status in (401, 402, 403): msg = T('Do not have valid authentication for feed %s') % feed logging.info(msg) return unicoder(msg) if status >= 500 and status <= 599: msg = T('Server side error (server code %s); could not get %s on %s') % (status, feed, uri) logging.info(msg) return unicoder(msg) entries = d.get('entries') if 'bozo_exception' in d and not entries: msg = str(d['bozo_exception']) if 'CERTIFICATE_VERIFY_FAILED' in msg: msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri) logging.error(msg) else: msg = T('Failed to retrieve RSS from %s: %s') % (uri, xml_name(msg)) logging.info(msg) return unicoder(msg) if not entries: msg = T('RSS Feed %s was empty') % uri logging.info(msg) if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() # Sort in the order the jobs came from the feed entries.sort(lambda x, y: jobs[x].get('order', 0) - jobs[y].get('order', 0)) order = 0 # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category, size = _get_link(uri, entry) except (AttributeError, IndexError): link = None category = u'' size = 0L logging.info(T('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info=True) return T('Incompatible feed') title = entry.title else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None title = jobs[link].get('title', '') size = jobs[link].get('size', 0L) if link: # Make sure spaces are quoted in the URL link = link.strip().replace(' ', '%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', title) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 if 'F' in reTypes: season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3] season = int_conv(season) episode = int_conv(episode) else: season = episode = 0 # Match against all filters until an positive or negative match logging.debug('Size %s for %s', size, title) for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break elif reTypes[n] == '<' and size and from_units(regexes[n]) < size: # "Size at most" : too large logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == '>' and size and from_units(regexes[n]) > size: # "Size at least" : too small logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == 'F' and not ep_match(season, episode, regexes[n]): # "Starting from SxxEyy", too early episode logging.debug('Filter requirement match on rule %d', n) result = False break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if notdefault(reCats[n]): myCat = reCats[n] elif category and not defCat: myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts(myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and dup_title(title): if cfg.no_dupes() == 1: logging.info("Ignoring duplicate job %s", title) continue else: myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, link, title, size, 'G', category, myCat, myPP, myScript, act, star, order, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, link, title, size, 'B', category, myCat, myPP, myScript, False, star, order, priority=myPrio, rule=str(n)) order += 1 # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return ''
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(T('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info=True) return T('Incorrect RSS feed description "%s"') % feed uris = feeds.uri() defCat = feeds.cat() import sabnzbd.api if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) if filter[3] in ('<', '>', 'F', 'S'): regexes.append(filter[4]) else: regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Read the RSS feed msg = None entries = None if readout: all_entries = [] for uri in uris: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) feed_parsed = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not feed_parsed: msg = T('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) status = feed_parsed.get('status', 999) if status in (401, 402, 403): msg = T('Do not have valid authentication for feed %s') % feed logging.info(msg) if status >= 500 and status <= 599: msg = T('Server side error (server code %s); could not get %s on %s') % (status, feed, uri) logging.info(msg) entries = feed_parsed.get('entries') if 'bozo_exception' in feed_parsed and not entries: msg = str(feed_parsed['bozo_exception']) if 'CERTIFICATE_VERIFY_FAILED' in msg: msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri) msg += ' - https://sabnzbd.org/certificate-errors' logging.error(msg) else: msg = T('Failed to retrieve RSS from %s: %s') % (uri, xml_name(msg)) logging.info(msg) if not entries: msg = T('RSS Feed %s was empty') % uri logging.info(msg) all_entries.extend(entries) entries = all_entries if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() # Sort in the order the jobs came from the feed entries.sort(lambda x, y: jobs[x].get('order', 0) - jobs[y].get('order', 0)) order = 0 # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category, size, age, season, episode = _get_link(uri, entry) except (AttributeError, IndexError): link = None category = u'' size = 0L age = None logging.info(T('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info=True) return T('Incompatible feed') title = entry.title # If there's multiple feeds, remove the duplicates based on title and size if len(uris) > 1: skip_job = False for job_link, job in jobs.items(): # Allow 5% size deviation because indexers might have small differences for same release if job.get('title') == title and link != job_link and (job.get('size')*0.95) < size < (job.get('size')*1.05): logging.info("Ignoring job %s from other feed", title) skip_job = True break if skip_job: continue else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None title = jobs[link].get('title', '') size = jobs[link].get('size', 0L) age = jobs[link].get('age') season = jobs[link].get('season', 0) episode = jobs[link].get('episode', 0) if link: # Make sure spaces are quoted in the URL link = link.strip().replace(' ', '%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', title) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 if ('F' in reTypes or 'S' in reTypes) and (not season or not episode): season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3] season = int_conv(season) episode = int_conv(episode) # Match against all filters until an positive or negative match logging.debug('Size %s for %s', size, title) for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break elif reTypes[n] == '<' and size and from_units(regexes[n]) < size: # "Size at most" : too large logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == '>' and size and from_units(regexes[n]) > size: # "Size at least" : too small logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == 'F' and not ep_match(season, episode, regexes[n]): # "Starting from SxxEyy", too early episode logging.debug('Filter requirement match on rule %d', n) result = False break elif reTypes[n] == 'S' and season and episode and ep_match(season, episode, regexes[n], title): logging.debug('Filter matched on rule %d', n) result = True break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if not result and defCat: # Apply Feed-category on non-matched items myCat = defCat elif result and notdefault(reCats[n]): # Use the matched info myCat = reCats[n] elif category and not defCat: # No result and no Feed-category myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts(myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and self.check_duplicate(title): if cfg.no_dupes() == 1: # Dupe-detection: Discard logging.info("Ignoring duplicate job %s", title) continue elif cfg.no_dupes() == 3: # Dupe-detection: Fail # We accept it so the Queue can send it to the History logging.info("Found duplicate job %s", title) else: # Dupe-detection: Pause myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, link, title, size, age, season, episode, 'G', category, myCat, myPP, myScript, act, star, order, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, link, title, size, age, season, episode, 'B', category, myCat, myPP, myScript, False, star, order, priority=myPrio, rule=str(n)) order += 1 # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return msg
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False def dup_title(title): """ Check if this title was in this or other feeds Return matching feed name """ title = title.lower() for fd in self.jobs: for lk in self.jobs[fd]: item = self.jobs[fd][lk] if item.get('status', ' ')[0] == 'D' and \ item.get('title', '').lower() == title: return fd return '' if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(Ta('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info = True) return T('Incorrect RSS feed description "%s"') % feed uri = feeds.uri() defCat = feeds.cat() if not notdefault(defCat): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Check for nzbs.org if 'nzbs.org/' in uri and not ('&dl=1' in uri): uri += '&dl=1' # Read the RSS feed msg = None entries = None if readout: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) d = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not d: msg = Ta('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) return unicoder(msg) status = d.get('status', 999) if status in (401, 402, 403): msg = Ta('Do not have valid authentication for feed %s') % feed logging.info(msg) return unicoder(msg) entries = d.get('entries') if 'bozo_exception' in d and not entries: msg = Ta('Failed to retrieve RSS from %s: %s') % (uri, xml_name(str(d['bozo_exception']))) logging.info(msg) return unicoder(msg) if not entries: msg = Ta('RSS Feed %s was empty') % uri logging.info(msg) if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() order = 0 # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category = _get_link(uri, entry) except (AttributeError, IndexError): link = None category = '' logging.info(Ta('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info = True) return T('Incompatible feed') category = latin1(category) # Make sure only latin-1 encodable characters occur atitle = latin1(entry.title) title = unicoder(atitle) else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None atitle = latin1(jobs[link].get('title', '')) title = unicoder(atitle) if link: # Make sure spaces are quoted in the URL if 'nzbclub.com' in link: link, path = os.path.split(link.strip()) link = '%s/%s' % (link, urllib.quote(path)) else: link = link.strip().replace(' ','%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', atitle) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 # Match against all filters until an postive or negative match for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if notdefault(reCats[n]): myCat = reCats[n] elif category and not defCat: myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts(myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and dup_title(title): if cfg.no_dupes() == 1: logging.info("Ignoring duplicate job %s", atitle) continue else: myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, link, title, 'G', category, myCat, myPP, myScript, act, star, order, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, link, title, 'B', category, myCat, myPP, myScript, False, star, order, priority=myPrio, rule=str(n)) order += 1 # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return ''
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False def dup_title(title): """ Check if this title was in this or other feeds Return matching feed name """ title = title.lower() for fd in self.jobs: for lk in self.jobs[fd]: item = self.jobs[fd][lk] if item.get('status', ' ')[0] == 'D' and \ item.get('title', '').lower() == title: return fd return '' if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(Ta('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info=True) return T('Incorrect RSS feed description "%s"') % feed uri = feeds.uri() defCat = feeds.cat() if not notdefault(defCat): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Check for nzbs.org if 'nzbs.org/' in uri and not ('&dl=1' in uri): uri += '&dl=1' # Read the RSS feed msg = None entries = None if readout: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) d = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not d: msg = Ta('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) return unicoder(msg) status = d.get('status', 999) if status in (401, 402, 403): msg = Ta('Do not have valid authentication for feed %s') % feed logging.info(msg) return unicoder(msg) entries = d.get('entries') if 'bozo_exception' in d and not entries: msg = Ta('Failed to retrieve RSS from %s: %s') % ( uri, xml_name(str(d['bozo_exception']))) logging.info(msg) return unicoder(msg) if not entries: msg = Ta('RSS Feed %s was empty') % uri logging.info(msg) if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() order = 0 # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category = _get_link(uri, entry) except (AttributeError, IndexError): link = None category = '' logging.info(Ta('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info=True) return T('Incompatible feed') category = latin1(category) # Make sure only latin-1 encodable characters occur atitle = latin1(entry.title) title = unicoder(atitle) else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None atitle = latin1(jobs[link].get('title', '')) title = unicoder(atitle) if link: # Make sure spaces are quoted in the URL if 'nzbclub.com' in link: link, path = os.path.split(link.strip()) link = '%s/%s' % (link, urllib.quote(path)) else: link = link.strip().replace(' ', '%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', atitle) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 # Match against all filters until an postive or negative match for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if notdefault(reCats[n]): myCat = reCats[n] elif category and not defCat: myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts( myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and dup_title(title): if cfg.no_dupes() == 1: logging.info("Ignoring duplicate job %s", atitle) continue else: myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, link, title, 'G', category, myCat, myPP, myScript, act, star, order, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, link, title, 'B', category, myCat, myPP, myScript, False, star, order, priority=myPrio, rule=str(n)) order += 1 # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return ''
def run(self): """ Process the queue (including waits and retries) """ from sabnzbd.nzbqueue import NzbQueue self.shutdown = False while not self.shutdown: time.sleep(5) (msgid, nzo) = self.queue.get() if self.shutdown or not msgid: break if nzo.wait and nzo.wait > time.time(): self.grab(msgid, nzo) continue logging.debug("Popping msgid %s", msgid) filename, data, newzbin_cat, nzo_info = _grabnzb(msgid) if filename and data: filename = name_fixer(filename) pp = nzo.pp script = nzo.script cat = nzo.cat if cat == '*' or not cat: cat = cat_convert(newzbin_cat) priority = nzo.priority nzbname = nzo.custom_name cat, pp, script, priority = cat_to_opts( cat, pp, script, priority) try: sabnzbd.nzbqueue.insert_future_nzo(nzo, filename, msgid, data, pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname, nzo_info=nzo_info) nzo.url = format_source_url(str(msgid)) except: logging.error(Ta('Failed to update newzbin job %s'), msgid) logging.info("Traceback: ", exc_info=True) NzbQueue.do.remove(nzo.nzo_id, False) msgid = None else: if filename: self.grab(msgid, nzo, float(filename)) else: # Fatal error, give up on this one bad_fetch(nzo, msgid, msg=nzo_info, retry=True) msgid = None if msgid: growler.send_notification(T('NZB added to queue'), filename, 'download') logging.debug('Stopping MSGIDGrabber')
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False if not feed: return "No such feed" newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(T('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info=True) return T('Incorrect RSS feed description "%s"') % feed uris = feeds.uri() defCat = feeds.cat() import sabnzbd.api if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for feed_filter in feeds.filters(): reCat = feed_filter[0] if defCat in ("", "*"): reCat = None reCats.append(reCat) rePPs.append(feed_filter[1]) reScripts.append(feed_filter[2]) reTypes.append(feed_filter[3]) if feed_filter[3] in ("<", ">", "F", "S"): regexes.append(feed_filter[4]) else: regexes.append(convert_filter(feed_filter[4])) rePrios.append(feed_filter[5]) reEnabled.append(feed_filter[6] != "0") regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add SABnzbd's custom User Agent feedparser.USER_AGENT = "SABnzbd/%s" % sabnzbd.__version__ # Read the RSS feed msg = None entries = None if readout: all_entries = [] for uri in uris: uri = uri.replace(" ", "%20") logging.debug("Running feedparser on %s", uri) feed_parsed = feedparser.parse(uri.replace("feed://", "http://")) logging.debug("Done parsing %s", uri) if not feed_parsed: msg = T("Failed to retrieve RSS from %s: %s") % (uri, "?") logging.info(msg) status = feed_parsed.get("status", 999) if status in (401, 402, 403): msg = T("Do not have valid authentication for feed %s") % uri logging.info(msg) if 500 <= status <= 599: msg = T("Server side error (server code %s); could not get %s on %s") % (status, feed, uri) logging.info(msg) entries = feed_parsed.get("entries") if "bozo_exception" in feed_parsed and not entries: msg = str(feed_parsed["bozo_exception"]) if "CERTIFICATE_VERIFY_FAILED" in msg: msg = T("Server %s uses an untrusted HTTPS certificate") % get_base_url(uri) msg += " - https://sabnzbd.org/certificate-errors" logging.error(msg) elif "href" in feed_parsed and feed_parsed["href"] != uri and "login" in feed_parsed["href"]: # Redirect to login page! msg = T("Do not have valid authentication for feed %s") % uri else: msg = T("Failed to retrieve RSS from %s: %s") % (uri, msg) logging.info(msg) if not entries and not msg: msg = T("RSS Feed %s was empty") % uri logging.info(msg) all_entries.extend(entries) entries = all_entries # In case of a new feed if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] # Error in readout or now new readout if readout: if not entries: return msg else: entries = jobs # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, infourl, category, size, age, season, episode = _get_link(entry) except (AttributeError, IndexError): logging.info(T("Incompatible feed") + " " + uri) logging.info("Traceback: ", exc_info=True) return T("Incompatible feed") title = entry.title # If there's multiple feeds, remove the duplicates based on title and size if len(uris) > 1: skip_job = False for job_link, job in jobs.items(): # Allow 5% size deviation because indexers might have small differences for same release if ( job.get("title") == title and link != job_link and (job.get("size") * 0.95) < size < (job.get("size") * 1.05) ): logging.info("Ignoring job %s from other feed", title) skip_job = True break if skip_job: continue else: link = entry infourl = jobs[link].get("infourl", "") category = jobs[link].get("orgcat", "") if category in ("", "*"): category = None title = jobs[link].get("title", "") size = jobs[link].get("size", 0) age = jobs[link].get("age") season = jobs[link].get("season", 0) episode = jobs[link].get("episode", 0) if link: # Make sure spaces are quoted in the URL link = link.strip().replace(" ", "%20") newlinks.append(link) if link in jobs: jobstat = jobs[link].get("status", " ")[0] else: jobstat = "N" if jobstat in "NGB" or (jobstat == "X" and readout): # Match this title against all filters logging.debug("Trying title %s", title) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 if ("F" in reTypes or "S" in reTypes) and (not season or not episode): season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3] # Match against all filters until an positive or negative match logging.debug("Size %s", size) for n in range(regcount): if reEnabled[n]: if category and reTypes[n] == "C": found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break elif reTypes[n] == "<" and size and from_units(regexes[n]) < size: # "Size at most" : too large logging.debug("Filter rejected on rule %d", n) result = False break elif reTypes[n] == ">" and size and from_units(regexes[n]) > size: # "Size at least" : too small logging.debug("Filter rejected on rule %d", n) result = False break elif reTypes[n] == "F" and not ep_match(season, episode, regexes[n]): # "Starting from SxxEyy", too early episode logging.debug("Filter requirement match on rule %d", n) result = False break elif ( reTypes[n] == "S" and season and episode and ep_match(season, episode, regexes[n], title) ): logging.debug("Filter matched on rule %d", n) result = True break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == "M" and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == "A": logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == "R": logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if not result and defCat: # Apply Feed-category on non-matched items myCat = defCat elif result and notdefault(reCats[n]): # Use the matched info myCat = reCats[n] elif category and not defCat: # No result and no Feed-category myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts(myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ""): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and self.check_duplicate(title): if cfg.no_dupes() == 1: # Dupe-detection: Discard logging.info("Ignoring duplicate job %s", title) continue elif cfg.no_dupes() == 3: # Dupe-detection: Fail # We accept it so the Queue can send it to the History logging.info("Found duplicate job %s", title) else: # Dupe-detection: Pause myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get("status", "").endswith("*") act = act or force star = first or jobs[link].get("status", "").endswith("*") else: star = first if result: _HandleLink( jobs, link, infourl, title, size, age, season, episode, "G", category, myCat, myPP, myScript, act, star, priority=myPrio, rule=n, ) if act: new_downloads.append(title) else: _HandleLink( jobs, link, infourl, title, size, age, season, episode, "B", category, myCat, myPP, myScript, False, star, priority=myPrio, rule=n, ) # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return msg