def run(self):
""" Run all the URI's and filters """
if not sabnzbd.PAUSED_ALL:
active = False
if self.next_run < time.time():
self.next_run = time.time() + cfg.rss_rate.get() * 60
feeds = config.get_rss()
for feed in feeds.keys():
try:
if feeds[feed].enable.get():
logging.info('Starting scheduled RSS read-out for "%s"', feed)
active = True
self.run_feed(feed, download=True, ignoreFirst=True)
# Wait 15 seconds, else sites may get irritated
for unused in xrange(15):
if self.shutdown:
return
else:
time.sleep(1.0)
except KeyError:
# Feed must have been deleted
pass
if active:
self.save()
logging.info('Finished scheduled RSS read-outs')
def __init__(self):
self.jobs = {}
self.next_run = time.time()
self.shutdown = False
try:
defined = config.get_rss().keys()
feeds = sabnzbd.load_admin(RSS_FILE_NAME)
if type(feeds) == type({}):
for feed in feeds:
if feed not in defined:
logging.debug('Dropping obsolete data for feed "%s"',
feed)
continue
self.jobs[feed] = {}
for link in feeds[feed]:
# Consistency check on data
try:
item = feeds[feed][link]
if not isinstance(item, dict) or not isinstance(
item.get('title'), unicode):
raise IndexError
self.jobs[feed][link] = item
except (KeyError, IndexError):
logging.info(
'Incorrect entry in %s detected, discarding %s',
RSS_FILE_NAME, item)
remove_obsolete(self.jobs[feed], self.jobs[feed].keys())
except IOError:
logging.debug('Cannot read file %s', RSS_FILE_NAME)
def run(self):
""" Run all the URI's and filters """
if not sabnzbd.PAUSED_ALL:
active = False
if self.next_run < time.time():
self.next_run = time.time() + cfg.rss_rate.get() * 60
feeds = config.get_rss()
for feed in feeds.keys():
try:
if feeds[feed].enable.get():
logging.info('Starting scheduled RSS read-out for "%s"', feed)
active = True
self.run_feed(feed, download=True, ignoreFirst=True)
# Wait 15 seconds, else sites may get irritated
for unused in xrange(15):
if self.shutdown:
return
else:
time.sleep(1.0)
except KeyError:
# Feed must have been deleted
pass
if active:
self.save()
logging.info('Finished scheduled RSS read-outs')
def run(self):
""" Run all the URI's and filters """
if not sabnzbd.PAUSED_ALL:
active = False
if self.next_run < time.time():
self.next_run = time.time() + cfg.rss_rate() * 60
feeds = config.get_rss()
try:
for feed in feeds:
if feeds[feed].enable():
logging.info(
'Starting scheduled RSS read-out for "%s"', feed)
active = True
self.run_feed(feed, download=True, ignoreFirst=True)
# Wait 15 seconds, else sites may get irritated
for unused in range(15):
if self.shutdown:
return
else:
time.sleep(1.0)
except (KeyError, RuntimeError):
# Feed must have been deleted
logging.info(
"RSS read-out crashed, feed must have been deleted or edited"
)
logging.debug("Traceback: ", exc_info=True)
pass
if active:
self.save()
logging.info("Finished scheduled RSS read-outs")
def __init__(self):
def check_str(p):
return p is None or p == '' or isinstance(p, basestring)
def check_int(p):
try:
int(p)
return True
except:
return False
self.jobs = {}
self.next_run = time.time()
self.shutdown = False
try:
defined = config.get_rss().keys()
feeds = sabnzbd.load_admin(RSS_FILE_NAME)
if type(feeds) == type({}):
for feed in feeds:
if feed not in defined:
logging.debug('Dropping obsolete data for feed "%s"', feed)
continue
self.jobs[feed] = {}
for link in feeds[feed]:
data = feeds[feed][link]
# Consistency check on data
try:
item = feeds[feed][link]
if not isinstance(item, dict) or not isinstance(item.get('title'), unicode):
raise IndexError
self.jobs[feed][link] = item
except (KeyError, IndexError):
logging.info('Incorrect entry in %s detected, discarding %s', RSS_FILE_NAME, item)
remove_obsolete(self.jobs[feed], self.jobs[feed].keys())
except IOError:
logging.debug('Cannot read file %s', RSS_FILE_NAME)
def ListUris():
""" Return list of all RSS uris """
uris = []
for uri in config.get_rss():
uris.append(uri)
return uris
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True):
""" Run the query for one URI and apply filters """
self.shutdown = False
def dup_title(title):
""" Check if this title was in this or other feeds
Return matching feed name
"""
title = title.lower()
for fd in self.jobs:
for lk in self.jobs[fd]:
item = self.jobs[fd][lk]
if item.get('status', ' ')[0] == 'D' and \
item.get('title', '').lower() == title:
return fd
return ''
if not feed:
return 'No such feed'
newlinks = []
new_downloads = []
# Preparations, get options
try:
feeds = config.get_rss()[feed]
except KeyError:
logging.error(T('Incorrect RSS feed description "%s"'), feed)
logging.info("Traceback: ", exc_info=True)
return T('Incorrect RSS feed description "%s"') % feed
uri = feeds.uri()
defCat = feeds.cat()
import sabnzbd.api
if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False):
defCat = None
defPP = feeds.pp()
if not notdefault(defPP):
defPP = None
defScript = feeds.script()
if not notdefault(defScript):
defScript = None
defPrio = feeds.priority()
if not notdefault(defPrio):
defPrio = None
# Preparations, convert filters to regex's
regexes = []
reTypes = []
reCats = []
rePPs = []
rePrios = []
reScripts = []
reEnabled = []
for filter in feeds.filters():
reCat = filter[0]
if defCat in ('', '*'):
reCat = None
reCats.append(reCat)
rePPs.append(filter[1])
reScripts.append(filter[2])
reTypes.append(filter[3])
if filter[3] in ('<', '>', 'F'):
regexes.append(filter[4])
else:
regexes.append(convert_filter(filter[4]))
rePrios.append(filter[5])
reEnabled.append(filter[6] != '0')
regcount = len(regexes)
# Set first if this is the very first scan of this URI
first = (feed not in self.jobs) and ignoreFirst
# Add sabnzbd's custom User Agent
feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__
# Check for nzbs.org
if 'nzbs.org/' in uri and '&dl=1' not in uri:
uri += '&dl=1'
# Read the RSS feed
msg = None
entries = None
if readout:
uri = uri.replace(' ', '%20')
logging.debug("Running feedparser on %s", uri)
d = feedparser.parse(uri.replace('feed://', 'http://'))
logging.debug("Done parsing %s", uri)
if not d:
msg = T('Failed to retrieve RSS from %s: %s') % (uri, '?')
logging.info(msg)
return unicoder(msg)
status = d.get('status', 999)
if status in (401, 402, 403):
msg = T('Do not have valid authentication for feed %s') % feed
logging.info(msg)
return unicoder(msg)
if status >= 500 and status <= 599:
msg = T('Server side error (server code %s); could not get %s on %s') % (status, feed, uri)
logging.info(msg)
return unicoder(msg)
entries = d.get('entries')
if 'bozo_exception' in d and not entries:
msg = str(d['bozo_exception'])
if 'CERTIFICATE_VERIFY_FAILED' in msg:
msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri)
logging.error(msg)
else:
msg = T('Failed to retrieve RSS from %s: %s') % (uri, xml_name(msg))
logging.info(msg)
return unicoder(msg)
if not entries:
msg = T('RSS Feed %s was empty') % uri
logging.info(msg)
if feed not in self.jobs:
self.jobs[feed] = {}
jobs = self.jobs[feed]
if readout:
if not entries:
return unicoder(msg)
else:
entries = jobs.keys()
# Sort in the order the jobs came from the feed
entries.sort(lambda x, y: jobs[x].get('order', 0) - jobs[y].get('order', 0))
order = 0
# Filter out valid new links
for entry in entries:
if self.shutdown:
return
if readout:
try:
link, category, size = _get_link(uri, entry)
except (AttributeError, IndexError):
link = None
category = u''
size = 0L
logging.info(T('Incompatible feed') + ' ' + uri)
logging.info("Traceback: ", exc_info=True)
return T('Incompatible feed')
title = entry.title
else:
link = entry
category = jobs[link].get('orgcat', '')
if category in ('', '*'):
category = None
title = jobs[link].get('title', '')
size = jobs[link].get('size', 0L)
if link:
# Make sure spaces are quoted in the URL
link = link.strip().replace(' ', '%20')
newlinks.append(link)
if link in jobs:
jobstat = jobs[link].get('status', ' ')[0]
else:
jobstat = 'N'
if jobstat in 'NGB' or (jobstat == 'X' and readout):
# Match this title against all filters
logging.debug('Trying title %s', title)
result = False
myCat = defCat
myPP = defPP
myScript = defScript
myPrio = defPrio
n = 0
if 'F' in reTypes:
season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3]
season = int_conv(season)
episode = int_conv(episode)
else:
season = episode = 0
# Match against all filters until an positive or negative match
logging.debug('Size %s for %s', size, title)
for n in xrange(regcount):
if reEnabled[n]:
if category and reTypes[n] == 'C':
found = re.search(regexes[n], category)
if not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
elif reTypes[n] == '<' and size and from_units(regexes[n]) < size:
# "Size at most" : too large
logging.debug('Filter rejected on rule %d', n)
result = False
break
elif reTypes[n] == '>' and size and from_units(regexes[n]) > size:
# "Size at least" : too small
logging.debug('Filter rejected on rule %d', n)
result = False
break
elif reTypes[n] == 'F' and not ep_match(season, episode, regexes[n]):
# "Starting from SxxEyy", too early episode
logging.debug('Filter requirement match on rule %d', n)
result = False
break
else:
if regexes[n]:
found = re.search(regexes[n], title)
else:
found = False
if reTypes[n] == 'M' and not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
if found and reTypes[n] == 'A':
logging.debug("Filter matched on rule %d", n)
result = True
break
if found and reTypes[n] == 'R':
logging.debug("Filter rejected on rule %d", n)
result = False
break
if len(reCats):
if notdefault(reCats[n]):
myCat = reCats[n]
elif category and not defCat:
myCat = cat_convert(category)
if myCat:
myCat, catPP, catScript, catPrio = cat_to_opts(myCat)
else:
myCat = catPP = catScript = catPrio = None
if notdefault(rePPs[n]):
myPP = rePPs[n]
elif not (reCats[n] or category):
myPP = catPP
if notdefault(reScripts[n]):
myScript = reScripts[n]
elif not (notdefault(reCats[n]) or category):
myScript = catScript
if rePrios[n] not in (str(DEFAULT_PRIORITY), ''):
myPrio = rePrios[n]
elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category):
myPrio = catPrio
if cfg.no_dupes() and dup_title(title):
if cfg.no_dupes() == 1:
logging.info("Ignoring duplicate job %s", title)
continue
else:
myPrio = DUP_PRIORITY
act = download and not first
if link in jobs:
act = act and not jobs[link].get('status', '').endswith('*')
act = act or force
star = first or jobs[link].get('status', '').endswith('*')
else:
star = first
if result:
_HandleLink(jobs, link, title, size, 'G', category, myCat, myPP, myScript,
act, star, order, priority=myPrio, rule=str(n))
if act:
new_downloads.append(title)
else:
_HandleLink(jobs, link, title, size, 'B', category, myCat, myPP, myScript,
False, star, order, priority=myPrio, rule=str(n))
order += 1
# Send email if wanted and not "forced"
if new_downloads and cfg.email_rss() and not force:
emailer.rss_mail(feed, new_downloads)
remove_obsolete(jobs, newlinks)
return ''
def __init__(self):
def check_str(p):
return p is None or p == '' or isinstance(p, basestring)
def check_int(p):
try:
int(p)
return True
except:
return False
self.jobs = {}
self.next_run = time.time()
try:
defined = config.get_rss().keys()
feeds = sabnzbd.load_admin(RSS_FILE_NAME)
if type(feeds) == type({}):
for feed in feeds:
if feed not in defined:
logging.debug('Dropping obsolete data for feed "%s"', feed)
continue
self.jobs[feed] = {}
for link in feeds[feed]:
data = feeds[feed][link]
if type(data) == type([]):
# Convert previous list-based store to dictionary
new = {}
try:
new['status'] = data[0]
new['title'] = data[1]
new['url'] = data[2]
new['cat'] = data[3]
new['pp'] = data[4]
new['script'] = data[5]
new['time'] = data[6]
new['prio'] = str(NORMAL_PRIORITY)
new['rule'] = 0
self.jobs[feed][link] = new
except IndexError:
del new
else:
# Consistency check on data
try:
item = feeds[feed][link]
if not isinstance(item, dict) or not isinstance(item.get('title'), unicode):
raise IndexError
if item.get('status', ' ')[0] not in ('D', 'G', 'B', 'X'):
item['status'] = 'X'
if not isinstance(item.get('url'), unicode):
item['url'] = ''
if not check_str(item.get('cat')):
item['cat'] = ''
if not check_str(item.get('orgcat')):
item['orgcat'] = ''
if not check_str(item.get('pp')):
item['pp'] = '3'
if not check_str(item.get('script')):
item['script'] = 'None'
if not check_str(item.get('prio')):
item['prio'] = '-100'
if not check_int(item.get('rule', 0)):
item['rule'] = 0
if not check_int(item.get('size', 0L)):
item['size'] = 0L
if not isinstance(item.get('time'), float):
item['time'] = time.time()
if not check_int(item.get('order', 0)):
item.get['order'] = 0
self.jobs[feed][link] = item
except (KeyError, IndexError):
logging.info('Incorrect entry in %s detected, discarding %s', RSS_FILE_NAME, item)
def ListUris():
""" Return list of all RSS uris """
uris = []
for uri in config.get_rss():
uris.append(uri)
return uris
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True):
""" Run the query for one URI and apply filters """
self.shutdown = False
if not feed:
return 'No such feed'
newlinks = []
new_downloads = []
# Preparations, get options
try:
feeds = config.get_rss()[feed]
except KeyError:
logging.error(T('Incorrect RSS feed description "%s"'), feed)
logging.info("Traceback: ", exc_info=True)
return T('Incorrect RSS feed description "%s"') % feed
uris = feeds.uri()
defCat = feeds.cat()
import sabnzbd.api
if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False):
defCat = None
defPP = feeds.pp()
if not notdefault(defPP):
defPP = None
defScript = feeds.script()
if not notdefault(defScript):
defScript = None
defPrio = feeds.priority()
if not notdefault(defPrio):
defPrio = None
# Preparations, convert filters to regex's
regexes = []
reTypes = []
reCats = []
rePPs = []
rePrios = []
reScripts = []
reEnabled = []
for filter in feeds.filters():
reCat = filter[0]
if defCat in ('', '*'):
reCat = None
reCats.append(reCat)
rePPs.append(filter[1])
reScripts.append(filter[2])
reTypes.append(filter[3])
if filter[3] in ('<', '>', 'F', 'S'):
regexes.append(filter[4])
else:
regexes.append(convert_filter(filter[4]))
rePrios.append(filter[5])
reEnabled.append(filter[6] != '0')
regcount = len(regexes)
# Set first if this is the very first scan of this URI
first = (feed not in self.jobs) and ignoreFirst
# Add sabnzbd's custom User Agent
feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__
# Read the RSS feed
msg = None
entries = None
if readout:
all_entries = []
for uri in uris:
uri = uri.replace(' ', '%20')
logging.debug("Running feedparser on %s", uri)
feed_parsed = feedparser.parse(uri.replace('feed://', 'http://'))
logging.debug("Done parsing %s", uri)
if not feed_parsed:
msg = T('Failed to retrieve RSS from %s: %s') % (uri, '?')
logging.info(msg)
status = feed_parsed.get('status', 999)
if status in (401, 402, 403):
msg = T('Do not have valid authentication for feed %s') % feed
logging.info(msg)
if status >= 500 and status <= 599:
msg = T('Server side error (server code %s); could not get %s on %s') % (status, feed, uri)
logging.info(msg)
entries = feed_parsed.get('entries')
if 'bozo_exception' in feed_parsed and not entries:
msg = str(feed_parsed['bozo_exception'])
if 'CERTIFICATE_VERIFY_FAILED' in msg:
msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri)
msg += ' - https://sabnzbd.org/certificate-errors'
logging.error(msg)
else:
msg = T('Failed to retrieve RSS from %s: %s') % (uri, xml_name(msg))
logging.info(msg)
if not entries:
msg = T('RSS Feed %s was empty') % uri
logging.info(msg)
all_entries.extend(entries)
entries = all_entries
if feed not in self.jobs:
self.jobs[feed] = {}
jobs = self.jobs[feed]
if readout:
if not entries:
return unicoder(msg)
else:
entries = jobs.keys()
# Sort in the order the jobs came from the feed
entries.sort(lambda x, y: jobs[x].get('order', 0) - jobs[y].get('order', 0))
order = 0
# Filter out valid new links
for entry in entries:
if self.shutdown:
return
if readout:
try:
link, category, size, age, season, episode = _get_link(uri, entry)
except (AttributeError, IndexError):
link = None
category = u''
size = 0L
age = None
logging.info(T('Incompatible feed') + ' ' + uri)
logging.info("Traceback: ", exc_info=True)
return T('Incompatible feed')
title = entry.title
# If there's multiple feeds, remove the duplicates based on title and size
if len(uris) > 1:
skip_job = False
for job_link, job in jobs.items():
# Allow 5% size deviation because indexers might have small differences for same release
if job.get('title') == title and link != job_link and (job.get('size')*0.95) < size < (job.get('size')*1.05):
logging.info("Ignoring job %s from other feed", title)
skip_job = True
break
if skip_job:
continue
else:
link = entry
category = jobs[link].get('orgcat', '')
if category in ('', '*'):
category = None
title = jobs[link].get('title', '')
size = jobs[link].get('size', 0L)
age = jobs[link].get('age')
season = jobs[link].get('season', 0)
episode = jobs[link].get('episode', 0)
if link:
# Make sure spaces are quoted in the URL
link = link.strip().replace(' ', '%20')
newlinks.append(link)
if link in jobs:
jobstat = jobs[link].get('status', ' ')[0]
else:
jobstat = 'N'
if jobstat in 'NGB' or (jobstat == 'X' and readout):
# Match this title against all filters
logging.debug('Trying title %s', title)
result = False
myCat = defCat
myPP = defPP
myScript = defScript
myPrio = defPrio
n = 0
if ('F' in reTypes or 'S' in reTypes) and (not season or not episode):
season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3]
season = int_conv(season)
episode = int_conv(episode)
# Match against all filters until an positive or negative match
logging.debug('Size %s for %s', size, title)
for n in xrange(regcount):
if reEnabled[n]:
if category and reTypes[n] == 'C':
found = re.search(regexes[n], category)
if not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
elif reTypes[n] == '<' and size and from_units(regexes[n]) < size:
# "Size at most" : too large
logging.debug('Filter rejected on rule %d', n)
result = False
break
elif reTypes[n] == '>' and size and from_units(regexes[n]) > size:
# "Size at least" : too small
logging.debug('Filter rejected on rule %d', n)
result = False
break
elif reTypes[n] == 'F' and not ep_match(season, episode, regexes[n]):
# "Starting from SxxEyy", too early episode
logging.debug('Filter requirement match on rule %d', n)
result = False
break
elif reTypes[n] == 'S' and season and episode and ep_match(season, episode, regexes[n], title):
logging.debug('Filter matched on rule %d', n)
result = True
break
else:
if regexes[n]:
found = re.search(regexes[n], title)
else:
found = False
if reTypes[n] == 'M' and not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
if found and reTypes[n] == 'A':
logging.debug("Filter matched on rule %d", n)
result = True
break
if found and reTypes[n] == 'R':
logging.debug("Filter rejected on rule %d", n)
result = False
break
if len(reCats):
if not result and defCat:
# Apply Feed-category on non-matched items
myCat = defCat
elif result and notdefault(reCats[n]):
# Use the matched info
myCat = reCats[n]
elif category and not defCat:
# No result and no Feed-category
myCat = cat_convert(category)
if myCat:
myCat, catPP, catScript, catPrio = cat_to_opts(myCat)
else:
myCat = catPP = catScript = catPrio = None
if notdefault(rePPs[n]):
myPP = rePPs[n]
elif not (reCats[n] or category):
myPP = catPP
if notdefault(reScripts[n]):
myScript = reScripts[n]
elif not (notdefault(reCats[n]) or category):
myScript = catScript
if rePrios[n] not in (str(DEFAULT_PRIORITY), ''):
myPrio = rePrios[n]
elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category):
myPrio = catPrio
if cfg.no_dupes() and self.check_duplicate(title):
if cfg.no_dupes() == 1:
# Dupe-detection: Discard
logging.info("Ignoring duplicate job %s", title)
continue
elif cfg.no_dupes() == 3:
# Dupe-detection: Fail
# We accept it so the Queue can send it to the History
logging.info("Found duplicate job %s", title)
else:
# Dupe-detection: Pause
myPrio = DUP_PRIORITY
act = download and not first
if link in jobs:
act = act and not jobs[link].get('status', '').endswith('*')
act = act or force
star = first or jobs[link].get('status', '').endswith('*')
else:
star = first
if result:
_HandleLink(jobs, link, title, size, age, season, episode, 'G', category, myCat, myPP, myScript,
act, star, order, priority=myPrio, rule=str(n))
if act:
new_downloads.append(title)
else:
_HandleLink(jobs, link, title, size, age, season, episode, 'B', category, myCat, myPP, myScript,
False, star, order, priority=myPrio, rule=str(n))
order += 1
# Send email if wanted and not "forced"
if new_downloads and cfg.email_rss() and not force:
emailer.rss_mail(feed, new_downloads)
remove_obsolete(jobs, newlinks)
return msg
def __init__(self):
def check_str(p):
return p is None or p == '' or isinstance(p, basestring)
def check_int(p):
try:
int(p)
return True
except:
return False
self.jobs = {}
self.next_run = time.time()
try:
defined = config.get_rss().keys()
feeds = sabnzbd.load_admin(RSS_FILE_NAME)
if type(feeds) == type({}):
for feed in feeds:
if feed not in defined:
logging.debug('Dropping obsolete data for feed "%s"',
feed)
continue
self.jobs[feed] = {}
for link in feeds[feed]:
data = feeds[feed][link]
if type(data) == type([]):
# Convert previous list-based store to dictionary
new = {}
try:
new['status'] = data[0]
new['title'] = data[1]
new['url'] = data[2]
new['cat'] = data[3]
new['pp'] = data[4]
new['script'] = data[5]
new['time'] = data[6]
new['prio'] = str(NORMAL_PRIORITY)
new['rule'] = 0
self.jobs[feed][link] = new
except IndexError:
del new
else:
# Consistency check on data
try:
item = feeds[feed][link]
if not isinstance(
item, dict) or not isinstance(
item.get('title'), unicode):
raise IndexError
if item.get('status',
' ')[0] not in ('D', 'G', 'B',
'X'):
item['status'] = 'X'
if not isinstance(item.get('url'), unicode):
item['url'] = ''
if not check_str(item.get('cat')):
item['cat'] = ''
if not check_str(item.get('orgcat')):
item['orgcat'] = ''
if not check_str(item.get('pp')):
item['pp'] = '3'
if not check_str(item.get('script')):
item['script'] = 'None'
if not check_str(item.get('prio')):
item['prio'] = '-100'
if not check_int(item.get('rule', 0)):
item['rule'] = 0
if not check_int(item.get('size', 0L)):
item['size'] = 0L
if not isinstance(item.get('time'), float):
item['time'] = time.time()
if not check_int(item.get('order', 0)):
item.get['order'] = 0
self.jobs[feed][link] = item
except (KeyError, IndexError):
logging.info(
'Incorrect entry in %s detected, discarding %s',
RSS_FILE_NAME, item)
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True):
""" Run the query for one URI and apply filters """
self.shutdown = False
if not feed:
return 'No such feed'
newlinks = []
new_downloads = []
# Preparations, get options
try:
feeds = config.get_rss()[feed]
except KeyError:
logging.error(T('Incorrect RSS feed description "%s"'), feed)
logging.info("Traceback: ", exc_info=True)
return T('Incorrect RSS feed description "%s"') % feed
uris = feeds.uri()
defCat = feeds.cat()
import sabnzbd.api
if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False):
defCat = None
defPP = feeds.pp()
if not notdefault(defPP):
defPP = None
defScript = feeds.script()
if not notdefault(defScript):
defScript = None
defPrio = feeds.priority()
if not notdefault(defPrio):
defPrio = None
# Preparations, convert filters to regex's
regexes = []
reTypes = []
reCats = []
rePPs = []
rePrios = []
reScripts = []
reEnabled = []
for filter in feeds.filters():
reCat = filter[0]
if defCat in ('', '*'):
reCat = None
reCats.append(reCat)
rePPs.append(filter[1])
reScripts.append(filter[2])
reTypes.append(filter[3])
if filter[3] in ('<', '>', 'F', 'S'):
regexes.append(filter[4])
else:
regexes.append(convert_filter(filter[4]))
rePrios.append(filter[5])
reEnabled.append(filter[6] != '0')
regcount = len(regexes)
# Set first if this is the very first scan of this URI
first = (feed not in self.jobs) and ignoreFirst
# Add sabnzbd's custom User Agent
feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__
# Read the RSS feed
msg = None
entries = None
if readout:
all_entries = []
for uri in uris:
uri = uri.replace(' ', '%20')
logging.debug("Running feedparser on %s", uri)
feed_parsed = feedparser.parse(uri.replace('feed://', 'http://'))
logging.debug("Done parsing %s", uri)
if not feed_parsed:
msg = T('Failed to retrieve RSS from %s: %s') % (uri, '?')
logging.info(msg)
status = feed_parsed.get('status', 999)
if status in (401, 402, 403):
msg = T('Do not have valid authentication for feed %s') % feed
logging.info(msg)
if 500 <= status <= 599:
msg = T('Server side error (server code %s); could not get %s on %s') % (status, feed, uri)
logging.info(msg)
entries = feed_parsed.get('entries')
if 'bozo_exception' in feed_parsed and not entries:
msg = str(feed_parsed['bozo_exception'])
if 'CERTIFICATE_VERIFY_FAILED' in msg:
msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri)
msg += ' - https://sabnzbd.org/certificate-errors'
logging.error(msg)
else:
msg = T('Failed to retrieve RSS from %s: %s') % (uri, xml_name(msg))
logging.info(msg)
if not entries:
msg = T('RSS Feed %s was empty') % uri
logging.info(msg)
all_entries.extend(entries)
entries = all_entries
# In case of a new feed
if feed not in self.jobs:
self.jobs[feed] = {}
jobs = self.jobs[feed]
# Error in readout or now new readout
if readout:
if not entries:
return unicoder(msg)
else:
entries = jobs.keys()
# Filter out valid new links
for entry in entries:
if self.shutdown:
return
if readout:
try:
link, category, size, age, season, episode = _get_link(entry)
except (AttributeError, IndexError):
logging.info(T('Incompatible feed') + ' ' + uri)
logging.info("Traceback: ", exc_info=True)
return T('Incompatible feed')
title = entry.title
# If there's multiple feeds, remove the duplicates based on title and size
if len(uris) > 1:
skip_job = False
for job_link, job in jobs.items():
# Allow 5% size deviation because indexers might have small differences for same release
if job.get('title') == title and link != job_link and (job.get('size')*0.95) < size < (job.get('size')*1.05):
logging.info("Ignoring job %s from other feed", title)
skip_job = True
break
if skip_job:
continue
else:
link = entry
category = jobs[link].get('orgcat', '')
if category in ('', '*'):
category = None
title = jobs[link].get('title', '')
size = jobs[link].get('size', 0L)
age = jobs[link].get('age')
season = jobs[link].get('season', 0)
episode = jobs[link].get('episode', 0)
if link:
# Make sure spaces are quoted in the URL
link = link.strip().replace(' ', '%20')
newlinks.append(link)
if link in jobs:
jobstat = jobs[link].get('status', ' ')[0]
else:
jobstat = 'N'
if jobstat in 'NGB' or (jobstat == 'X' and readout):
# Match this title against all filters
logging.debug('Trying title %s', title)
result = False
myCat = defCat
myPP = defPP
myScript = defScript
myPrio = defPrio
n = 0
if ('F' in reTypes or 'S' in reTypes) and (not season or not episode):
season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3]
# Match against all filters until an positive or negative match
logging.debug('Size %s', size)
for n in xrange(regcount):
if reEnabled[n]:
if category and reTypes[n] == 'C':
found = re.search(regexes[n], category)
if not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
elif reTypes[n] == '<' and size and from_units(regexes[n]) < size:
# "Size at most" : too large
logging.debug('Filter rejected on rule %d', n)
result = False
break
elif reTypes[n] == '>' and size and from_units(regexes[n]) > size:
# "Size at least" : too small
logging.debug('Filter rejected on rule %d', n)
result = False
break
elif reTypes[n] == 'F' and not ep_match(season, episode, regexes[n]):
# "Starting from SxxEyy", too early episode
logging.debug('Filter requirement match on rule %d', n)
result = False
break
elif reTypes[n] == 'S' and season and episode and ep_match(season, episode, regexes[n], title):
logging.debug('Filter matched on rule %d', n)
result = True
break
else:
if regexes[n]:
found = re.search(regexes[n], title)
else:
found = False
if reTypes[n] == 'M' and not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
if found and reTypes[n] == 'A':
logging.debug("Filter matched on rule %d", n)
result = True
break
if found and reTypes[n] == 'R':
logging.debug("Filter rejected on rule %d", n)
result = False
break
if len(reCats):
if not result and defCat:
# Apply Feed-category on non-matched items
myCat = defCat
elif result and notdefault(reCats[n]):
# Use the matched info
myCat = reCats[n]
elif category and not defCat:
# No result and no Feed-category
myCat = cat_convert(category)
if myCat:
myCat, catPP, catScript, catPrio = cat_to_opts(myCat)
else:
myCat = catPP = catScript = catPrio = None
if notdefault(rePPs[n]):
myPP = rePPs[n]
elif not (reCats[n] or category):
myPP = catPP
if notdefault(reScripts[n]):
myScript = reScripts[n]
elif not (notdefault(reCats[n]) or category):
myScript = catScript
if rePrios[n] not in (str(DEFAULT_PRIORITY), ''):
myPrio = rePrios[n]
elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category):
myPrio = catPrio
if cfg.no_dupes() and self.check_duplicate(title):
if cfg.no_dupes() == 1:
# Dupe-detection: Discard
logging.info("Ignoring duplicate job %s", title)
continue
elif cfg.no_dupes() == 3:
# Dupe-detection: Fail
# We accept it so the Queue can send it to the History
logging.info("Found duplicate job %s", title)
else:
# Dupe-detection: Pause
myPrio = DUP_PRIORITY
act = download and not first
if link in jobs:
act = act and not jobs[link].get('status', '').endswith('*')
act = act or force
star = first or jobs[link].get('status', '').endswith('*')
else:
star = first
if result:
_HandleLink(jobs, feed, link, title, size, age, season, episode, 'G', category, myCat, myPP,
myScript, act, star, priority=myPrio, rule=str(n))
if act:
new_downloads.append(title)
else:
_HandleLink(jobs, feed, link, title, size, age, season, episode, 'B', category, myCat, myPP,
myScript, False, star, priority=myPrio, rule=str(n))
# Send email if wanted and not "forced"
if new_downloads and cfg.email_rss() and not force:
emailer.rss_mail(feed, new_downloads)
remove_obsolete(jobs, newlinks)
return msg
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True):
""" Run the query for one URI and apply filters """
self.shutdown = False
if not feed:
return "No such feed"
newlinks = []
new_downloads = []
# Preparations, get options
try:
feeds = config.get_rss()[feed]
except KeyError:
logging.error(T('Incorrect RSS feed description "%s"'), feed)
logging.info("Traceback: ", exc_info=True)
return T('Incorrect RSS feed description "%s"') % feed
uris = feeds.uri()
defCat = feeds.cat()
import sabnzbd.api
if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False):
defCat = None
defPP = feeds.pp()
if not notdefault(defPP):
defPP = None
defScript = feeds.script()
if not notdefault(defScript):
defScript = None
defPrio = feeds.priority()
if not notdefault(defPrio):
defPrio = None
# Preparations, convert filters to regex's
regexes = []
reTypes = []
reCats = []
rePPs = []
rePrios = []
reScripts = []
reEnabled = []
for feed_filter in feeds.filters():
reCat = feed_filter[0]
if defCat in ("", "*"):
reCat = None
reCats.append(reCat)
rePPs.append(feed_filter[1])
reScripts.append(feed_filter[2])
reTypes.append(feed_filter[3])
if feed_filter[3] in ("<", ">", "F", "S"):
regexes.append(feed_filter[4])
else:
regexes.append(convert_filter(feed_filter[4]))
rePrios.append(feed_filter[5])
reEnabled.append(feed_filter[6] != "0")
regcount = len(regexes)
# Set first if this is the very first scan of this URI
first = (feed not in self.jobs) and ignoreFirst
# Add SABnzbd's custom User Agent
feedparser.USER_AGENT = "SABnzbd/%s" % sabnzbd.__version__
# Read the RSS feed
msg = None
entries = None
if readout:
all_entries = []
for uri in uris:
uri = uri.replace(" ", "%20")
logging.debug("Running feedparser on %s", uri)
feed_parsed = feedparser.parse(uri.replace("feed://", "http://"))
logging.debug("Done parsing %s", uri)
if not feed_parsed:
msg = T("Failed to retrieve RSS from %s: %s") % (uri, "?")
logging.info(msg)
status = feed_parsed.get("status", 999)
if status in (401, 402, 403):
msg = T("Do not have valid authentication for feed %s") % uri
logging.info(msg)
if 500 <= status <= 599:
msg = T("Server side error (server code %s); could not get %s on %s") % (status, feed, uri)
logging.info(msg)
entries = feed_parsed.get("entries")
if "bozo_exception" in feed_parsed and not entries:
msg = str(feed_parsed["bozo_exception"])
if "CERTIFICATE_VERIFY_FAILED" in msg:
msg = T("Server %s uses an untrusted HTTPS certificate") % get_base_url(uri)
msg += " - https://sabnzbd.org/certificate-errors"
logging.error(msg)
elif "href" in feed_parsed and feed_parsed["href"] != uri and "login" in feed_parsed["href"]:
# Redirect to login page!
msg = T("Do not have valid authentication for feed %s") % uri
else:
msg = T("Failed to retrieve RSS from %s: %s") % (uri, msg)
logging.info(msg)
if not entries and not msg:
msg = T("RSS Feed %s was empty") % uri
logging.info(msg)
all_entries.extend(entries)
entries = all_entries
# In case of a new feed
if feed not in self.jobs:
self.jobs[feed] = {}
jobs = self.jobs[feed]
# Error in readout or now new readout
if readout:
if not entries:
return msg
else:
entries = jobs
# Filter out valid new links
for entry in entries:
if self.shutdown:
return
if readout:
try:
link, infourl, category, size, age, season, episode = _get_link(entry)
except (AttributeError, IndexError):
logging.info(T("Incompatible feed") + " " + uri)
logging.info("Traceback: ", exc_info=True)
return T("Incompatible feed")
title = entry.title
# If there's multiple feeds, remove the duplicates based on title and size
if len(uris) > 1:
skip_job = False
for job_link, job in jobs.items():
# Allow 5% size deviation because indexers might have small differences for same release
if (
job.get("title") == title
and link != job_link
and (job.get("size") * 0.95) < size < (job.get("size") * 1.05)
):
logging.info("Ignoring job %s from other feed", title)
skip_job = True
break
if skip_job:
continue
else:
link = entry
infourl = jobs[link].get("infourl", "")
category = jobs[link].get("orgcat", "")
if category in ("", "*"):
category = None
title = jobs[link].get("title", "")
size = jobs[link].get("size", 0)
age = jobs[link].get("age")
season = jobs[link].get("season", 0)
episode = jobs[link].get("episode", 0)
if link:
# Make sure spaces are quoted in the URL
link = link.strip().replace(" ", "%20")
newlinks.append(link)
if link in jobs:
jobstat = jobs[link].get("status", " ")[0]
else:
jobstat = "N"
if jobstat in "NGB" or (jobstat == "X" and readout):
# Match this title against all filters
logging.debug("Trying title %s", title)
result = False
myCat = defCat
myPP = defPP
myScript = defScript
myPrio = defPrio
n = 0
if ("F" in reTypes or "S" in reTypes) and (not season or not episode):
season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3]
# Match against all filters until an positive or negative match
logging.debug("Size %s", size)
for n in range(regcount):
if reEnabled[n]:
if category and reTypes[n] == "C":
found = re.search(regexes[n], category)
if not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
elif reTypes[n] == "<" and size and from_units(regexes[n]) < size:
# "Size at most" : too large
logging.debug("Filter rejected on rule %d", n)
result = False
break
elif reTypes[n] == ">" and size and from_units(regexes[n]) > size:
# "Size at least" : too small
logging.debug("Filter rejected on rule %d", n)
result = False
break
elif reTypes[n] == "F" and not ep_match(season, episode, regexes[n]):
# "Starting from SxxEyy", too early episode
logging.debug("Filter requirement match on rule %d", n)
result = False
break
elif (
reTypes[n] == "S"
and season
and episode
and ep_match(season, episode, regexes[n], title)
):
logging.debug("Filter matched on rule %d", n)
result = True
break
else:
if regexes[n]:
found = re.search(regexes[n], title)
else:
found = False
if reTypes[n] == "M" and not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
if found and reTypes[n] == "A":
logging.debug("Filter matched on rule %d", n)
result = True
break
if found and reTypes[n] == "R":
logging.debug("Filter rejected on rule %d", n)
result = False
break
if len(reCats):
if not result and defCat:
# Apply Feed-category on non-matched items
myCat = defCat
elif result and notdefault(reCats[n]):
# Use the matched info
myCat = reCats[n]
elif category and not defCat:
# No result and no Feed-category
myCat = cat_convert(category)
if myCat:
myCat, catPP, catScript, catPrio = cat_to_opts(myCat)
else:
myCat = catPP = catScript = catPrio = None
if notdefault(rePPs[n]):
myPP = rePPs[n]
elif not (reCats[n] or category):
myPP = catPP
if notdefault(reScripts[n]):
myScript = reScripts[n]
elif not (notdefault(reCats[n]) or category):
myScript = catScript
if rePrios[n] not in (str(DEFAULT_PRIORITY), ""):
myPrio = rePrios[n]
elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category):
myPrio = catPrio
if cfg.no_dupes() and self.check_duplicate(title):
if cfg.no_dupes() == 1:
# Dupe-detection: Discard
logging.info("Ignoring duplicate job %s", title)
continue
elif cfg.no_dupes() == 3:
# Dupe-detection: Fail
# We accept it so the Queue can send it to the History
logging.info("Found duplicate job %s", title)
else:
# Dupe-detection: Pause
myPrio = DUP_PRIORITY
act = download and not first
if link in jobs:
act = act and not jobs[link].get("status", "").endswith("*")
act = act or force
star = first or jobs[link].get("status", "").endswith("*")
else:
star = first
if result:
_HandleLink(
jobs,
link,
infourl,
title,
size,
age,
season,
episode,
"G",
category,
myCat,
myPP,
myScript,
act,
star,
priority=myPrio,
rule=n,
)
if act:
new_downloads.append(title)
else:
_HandleLink(
jobs,
link,
infourl,
title,
size,
age,
season,
episode,
"B",
category,
myCat,
myPP,
myScript,
False,
star,
priority=myPrio,
rule=n,
)
# Send email if wanted and not "forced"
if new_downloads and cfg.email_rss() and not force:
emailer.rss_mail(feed, new_downloads)
remove_obsolete(jobs, newlinks)
return msg
def buildMenu(self):
# logging.info("building menu")
status_bar = NSStatusBar.systemStatusBar()
self.status_item = status_bar.statusItemWithLength_(NSVariableStatusItemLength)
for icon in status_icons:
icon_path = status_icons[icon]
if hasattr(sys, "frozen"):
# Path is modified for the binary
icon_path = os.path.join(os.path.dirname(sys.executable), "..", "Resources", status_icons[icon])
self.icons[icon] = NSImage.alloc().initByReferencingFile_(icon_path)
if sabnzbd.DARWIN_VERSION > 9:
# Support for Yosemite Dark Mode
self.icons[icon].setTemplate_(YES)
self.status_item.setImage_(self.icons["idle"])
self.status_item.setAlternateImage_(self.icons["clicked"])
self.status_item.setHighlightMode_(1)
self.status_item.setToolTip_("SABnzbd")
self.status_item.setEnabled_(YES)
# Wait for translated texts to be loaded
while not sabnzbd.WEBUI_READY and not sabnzbd.SABSTOP:
time.sleep(0.5)
# Variables
self.state = "Idle"
self.speed = 0
self.version_notify = 1
self.status_removed = 0
# Menu construction
self.menu = NSMenu.alloc().init()
# Warnings Item
self.warnings_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("Warnings"), "openBrowserAction:", ""
)
self.warnings_menu_item.setHidden_(YES)
self.menu.addItem_(self.warnings_menu_item)
# State Item
self.state_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("Idle"), "openBrowserAction:", ""
)
self.menu.addItem_(self.state_menu_item)
# Queue Item
self.queue_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("Queue"), "openBrowserAction:", ""
)
self.menu.addItem_(self.queue_menu_item)
# Purge Queue Item
self.purgequeue_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("Purge Queue"), "purgeAction:", ""
)
self.purgequeue_menu_item.setRepresentedObject_("queue")
self.purgequeue_menu_item.setAlternate_(YES)
self.purgequeue_menu_item.setKeyEquivalentModifierMask_(NSAlternateKeyMask)
self.menu.addItem_(self.purgequeue_menu_item)
# History Item
self.history_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("History"), "openBrowserAction:", ""
)
self.menu.addItem_(self.history_menu_item)
# Purge History Item
self.purgehistory_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("Purge History"), "purgeAction:", ""
)
self.purgehistory_menu_item.setRepresentedObject_("history")
self.purgehistory_menu_item.setAlternate_(YES)
self.purgehistory_menu_item.setKeyEquivalentModifierMask_(NSAlternateKeyMask)
self.menu.addItem_(self.purgehistory_menu_item)
self.menu.addItem_(NSMenuItem.separatorItem())
# Limit Speed Item & Submenu
self.speed_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(T("Limit Speed"), "", "")
self.menu_speed = NSMenu.alloc().init()
for speed in range(10, 101, 10):
menu_speed_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
"%s%%" % speed, "speedlimitAction:", ""
)
menu_speed_item.setRepresentedObject_("%s" % speed)
self.menu_speed.addItem_(menu_speed_item)
self.speed_menu_item.setSubmenu_(self.menu_speed)
self.menu.addItem_(self.speed_menu_item)
# Pause Item & Submenu
self.pause_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(T("Pause"), "pauseAction:", "")
self.pause_menu_item.setRepresentedObject_("0")
self.menu_pause = NSMenu.alloc().init()
for i in range(6):
menu_pause_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
"%s %s" % ((i + 1) * 10, T("min")), "pauseAction:", ""
)
menu_pause_item.setRepresentedObject_("%s" % ((i + 1) * 10))
self.menu_pause.addItem_(menu_pause_item)
self.pause_menu_item.setSubmenu_(self.menu_pause)
self.menu.addItem_(self.pause_menu_item)
# Resume Item
self.resume_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(T("Resume"), "resumeAction:", "")
self.resume_menu_item.setHidden_(YES)
self.menu.addItem_(self.resume_menu_item)
# Watched folder Item
if sabnzbd.cfg.dirscan_dir():
self.watched_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("Scan watched folder"), "watchedFolderAction:", ""
)
self.menu.addItem_(self.watched_menu_item)
# Read RSS feeds
if config.get_rss():
self.rss_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("Read all RSS feeds"), "rssAction:", ""
)
self.menu.addItem_(self.rss_menu_item)
self.menu.addItem_(NSMenuItem.separatorItem())
# Complete Folder Item
self.completefolder_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("Complete Folder"), "openFolderAction:", ""
)
self.completefolder_menu_item.setRepresentedObject_(sabnzbd.cfg.complete_dir.get_path())
self.menu.addItem_(self.completefolder_menu_item)
# Incomplete Folder Item
self.incompletefolder_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(
T("Incomplete Folder"), "openFolderAction:", ""
)
self.incompletefolder_menu_item.setRepresentedObject_(sabnzbd.cfg.download_dir.get_path())
self.menu.addItem_(self.incompletefolder_menu_item)
self.menu.addItem_(NSMenuItem.separatorItem())
# Set diagnostic menu
self.diagnostic_menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(T("Troubleshoot"), "", "")
self.menu_diagnostic = NSMenu.alloc().init()
diag_items = (
(T("Restart"), "restartAction:"),
(T("Restart") + " - 127.0.0.1:8080", "restartSafeHost:"),
(T("Restart without login"), "restartNoLogin:"******"")
menu_diag_item.setRepresentedObject_(item[0])
self.menu_diagnostic.addItem_(menu_diag_item)
self.diagnostic_menu_item.setSubmenu_(self.menu_diagnostic)
self.menu.addItem_(self.diagnostic_menu_item)
# Quit Item
menu_item = NSMenuItem.alloc().initWithTitle_action_keyEquivalent_(T("Quit"), "terminate:", "")
self.menu.addItem_(menu_item)
# Add menu to Status Item
self.status_item.setMenu_(self.menu)
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True):
""" Run the query for one URI and apply filters """
self.shutdown = False
def dup_title(title):
""" Check if this title was in this or other feeds
Return matching feed name
"""
title = title.lower()
for fd in self.jobs:
for lk in self.jobs[fd]:
item = self.jobs[fd][lk]
if item.get('status', ' ')[0] == 'D' and \
item.get('title', '').lower() == title:
return fd
return ''
if not feed:
return 'No such feed'
newlinks = []
new_downloads = []
# Preparations, get options
try:
feeds = config.get_rss()[feed]
except KeyError:
logging.error(Ta('Incorrect RSS feed description "%s"'), feed)
logging.info("Traceback: ", exc_info = True)
return T('Incorrect RSS feed description "%s"') % feed
uri = feeds.uri()
defCat = feeds.cat()
if not notdefault(defCat):
defCat = None
defPP = feeds.pp()
if not notdefault(defPP):
defPP = None
defScript = feeds.script()
if not notdefault(defScript):
defScript = None
defPrio = feeds.priority()
if not notdefault(defPrio):
defPrio = None
# Preparations, convert filters to regex's
regexes = []
reTypes = []
reCats = []
rePPs = []
rePrios = []
reScripts = []
reEnabled = []
for filter in feeds.filters():
reCat = filter[0]
if defCat in ('', '*'):
reCat = None
reCats.append(reCat)
rePPs.append(filter[1])
reScripts.append(filter[2])
reTypes.append(filter[3])
regexes.append(convert_filter(filter[4]))
rePrios.append(filter[5])
reEnabled.append(filter[6] != '0')
regcount = len(regexes)
# Set first if this is the very first scan of this URI
first = (feed not in self.jobs) and ignoreFirst
# Add sabnzbd's custom User Agent
feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__
# Check for nzbs.org
if 'nzbs.org/' in uri and not ('&dl=1' in uri):
uri += '&dl=1'
# Read the RSS feed
msg = None
entries = None
if readout:
uri = uri.replace(' ', '%20')
logging.debug("Running feedparser on %s", uri)
d = feedparser.parse(uri.replace('feed://', 'http://'))
logging.debug("Done parsing %s", uri)
if not d:
msg = Ta('Failed to retrieve RSS from %s: %s') % (uri, '?')
logging.info(msg)
return unicoder(msg)
status = d.get('status', 999)
if status in (401, 402, 403):
msg = Ta('Do not have valid authentication for feed %s') % feed
logging.info(msg)
return unicoder(msg)
entries = d.get('entries')
if 'bozo_exception' in d and not entries:
msg = Ta('Failed to retrieve RSS from %s: %s') % (uri, xml_name(str(d['bozo_exception'])))
logging.info(msg)
return unicoder(msg)
if not entries:
msg = Ta('RSS Feed %s was empty') % uri
logging.info(msg)
if feed not in self.jobs:
self.jobs[feed] = {}
jobs = self.jobs[feed]
if readout:
if not entries:
return unicoder(msg)
else:
entries = jobs.keys()
order = 0
# Filter out valid new links
for entry in entries:
if self.shutdown: return
if readout:
try:
link, category = _get_link(uri, entry)
except (AttributeError, IndexError):
link = None
category = ''
logging.info(Ta('Incompatible feed') + ' ' + uri)
logging.info("Traceback: ", exc_info = True)
return T('Incompatible feed')
category = latin1(category)
# Make sure only latin-1 encodable characters occur
atitle = latin1(entry.title)
title = unicoder(atitle)
else:
link = entry
category = jobs[link].get('orgcat', '')
if category in ('', '*'):
category = None
atitle = latin1(jobs[link].get('title', ''))
title = unicoder(atitle)
if link:
# Make sure spaces are quoted in the URL
if 'nzbclub.com' in link:
link, path = os.path.split(link.strip())
link = '%s/%s' % (link, urllib.quote(path))
else:
link = link.strip().replace(' ','%20')
newlinks.append(link)
if link in jobs:
jobstat = jobs[link].get('status', ' ')[0]
else:
jobstat = 'N'
if jobstat in 'NGB' or (jobstat == 'X' and readout):
# Match this title against all filters
logging.debug('Trying title %s', atitle)
result = False
myCat = defCat
myPP = defPP
myScript = defScript
myPrio = defPrio
n = 0
# Match against all filters until an postive or negative match
for n in xrange(regcount):
if reEnabled[n]:
if category and reTypes[n] == 'C':
found = re.search(regexes[n], category)
if not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
else:
if regexes[n]:
found = re.search(regexes[n], title)
else:
found = False
if reTypes[n] == 'M' and not found:
logging.debug("Filter rejected on rule %d", n)
result = False
break
if found and reTypes[n] == 'A':
logging.debug("Filter matched on rule %d", n)
result = True
break
if found and reTypes[n] == 'R':
logging.debug("Filter rejected on rule %d", n)
result = False
break
if len(reCats):
if notdefault(reCats[n]):
myCat = reCats[n]
elif category and not defCat:
myCat = cat_convert(category)
if myCat:
myCat, catPP, catScript, catPrio = cat_to_opts(myCat)
else:
myCat = catPP = catScript = catPrio = None
if notdefault(rePPs[n]):
myPP = rePPs[n]
elif not (reCats[n] or category):
myPP = catPP
if notdefault(reScripts[n]):
myScript = reScripts[n]
elif not (notdefault(reCats[n]) or category):
myScript = catScript
if rePrios[n] not in (str(DEFAULT_PRIORITY), ''):
myPrio = rePrios[n]
elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category):
myPrio = catPrio
if cfg.no_dupes() and dup_title(title):
if cfg.no_dupes() == 1:
logging.info("Ignoring duplicate job %s", atitle)
continue
else:
myPrio = DUP_PRIORITY
act = download and not first
if link in jobs:
act = act and not jobs[link].get('status', '').endswith('*')
act = act or force
star = first or jobs[link].get('status', '').endswith('*')
else:
star = first
if result:
_HandleLink(jobs, link, title, 'G', category, myCat, myPP, myScript,
act, star, order, priority=myPrio, rule=str(n))
if act:
new_downloads.append(title)
else:
_HandleLink(jobs, link, title, 'B', category, myCat, myPP, myScript,
False, star, order, priority=myPrio, rule=str(n))
order += 1
# Send email if wanted and not "forced"
if new_downloads and cfg.email_rss() and not force:
emailer.rss_mail(feed, new_downloads)
remove_obsolete(jobs, newlinks)
return ''
def __init__(self):
def check_str(p):
return p is None or p == '' or isinstance(p, str)
def check_int(p):
try:
int(p)
return True
except:
return False
self.jobs = {}
try:
defined = config.get_rss().keys()
feeds = sabnzbd.load_admin(RSS_FILE_NAME)
if type(feeds) == type({}):
for feed in feeds:
if feed not in defined:
logging.debug('Dropping obsolete data for feed "%s"', feed)
continue
self.jobs[feed] = {}
for link in feeds[feed]:
data = feeds[feed][link]
if type(data) == type([]):
# Convert previous list-based store to dictionary
new = {}
try:
new['status'] = data[0]
new['title'] = data[1]
new['url'] = data[2]
new['cat'] = data[3]
new['pp'] = data[4]
new['script'] = data[5]
new['time'] = data[6]
new['prio'] = str(NORMAL_PRIORITY)
new['rule'] = 0
self.jobs[feed][link] = new
except IndexError:
del new
else:
# Consistency check on data
try:
item = feeds[feed][link]
if not isinstance(item, dict) or not isinstance(item.get('title'), unicode):
raise IndexError
if item.get('status', ' ')[0] not in ('D', 'G', 'B', 'X'):
item['status'] = 'X'
if not isinstance(item.get('url'), unicode): item['url'] = ''
item['url'] = item['url'].replace('www.newzbin.com', cfg.newzbin_url())
if not check_str(item.get('cat')): item['cat'] = ''
if not check_str(item.get('orgcat')): item['orgcat'] = ''
if not check_str(item.get('pp')): item['pp'] = '3'
if not check_str(item.get('script')): item['script'] = 'None'
if not check_str(item.get('prio')): item['prio'] = '-100'
if not check_int(item.get('rule', 0)): item['rule'] = 0
if not isinstance(item.get('time'), float): item['time'] = time.time()
if not check_int(item.get('order', 0)): item.get['order'] = 0
self.jobs[feed][link] = item
except (KeyError, IndexError):
logging.info('Incorrect entry in %s detected, discarding %s', RSS_FILE_NAME, item)
remove_obsolete(self.jobs[feed], self.jobs[feed].keys())
except IOError:
logging.debug('Cannot read file %s', RSS_FILE_NAME)
# jobs is a NAME-indexed dictionary
# Each element is link-indexed dictionary
# Each element is another dictionary:
# status : 'D', 'G', 'B', 'X' (downloaded, good-match, bad-match, obsolete)
# '*' added means: from the initial batch
# '-' added to 'D' means downloaded, but not displayed anymore
# title : Title
# url : URL or MsgId
# cat : category
# orgcat : category as read from feed
# pp : pp
# script : script
# prio : priority
# time : timestamp (used for time-based clean-up)
# order : order in the RSS feed
self.shutdown = False
def run_feed(self,
feed=None,
download=False,
ignoreFirst=False,
force=False,
readout=True):
""" Run the query for one URI and apply filters """
self.shutdown = False
def dup_title(title):
""" Check if this title was in this or other feeds
Return matching feed name
"""
title = title.lower()
for fd in self.jobs:
for lk in self.jobs[fd]:
item = self.jobs[fd][lk]
if item.get('status', ' ')[0] == 'D' and \
item.get('title', '').lower() == title:
return fd
return ''
if not feed:
return 'No such feed'
newlinks = []
new_downloads = []
# Preparations, get options
try:
feeds = config.get_rss()[feed]
except KeyError:
logging.error(Ta('Incorrect RSS feed description "%s"'), feed)
logging.info("Traceback: ", exc_info=True)
return T('Incorrect RSS feed description "%s"') % feed
uri = feeds.uri()
defCat = feeds.cat()
if not notdefault(defCat):
defCat = None
defPP = feeds.pp()
if not notdefault(defPP):
defPP = None
defScript = feeds.script()
if not notdefault(defScript):
defScript = None
defPrio = feeds.priority()
if not notdefault(defPrio):
defPrio = None
# Preparations, convert filters to regex's
regexes = []
reTypes = []
reCats = []
rePPs = []
rePrios = []
reScripts = []
reEnabled = []
for filter in feeds.filters():
reCat = filter[0]
if defCat in ('', '*'):
reCat = None
reCats.append(reCat)
rePPs.append(filter[1])
reScripts.append(filter[2])
reTypes.append(filter[3])
regexes.append(convert_filter(filter[4]))
rePrios.append(filter[5])
reEnabled.append(filter[6] != '0')
regcount = len(regexes)
# Set first if this is the very first scan of this URI
first = (feed not in self.jobs) and ignoreFirst
# Add sabnzbd's custom User Agent
feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__
# Check for nzbs.org
if 'nzbs.org/' in uri and not ('&dl=1' in uri):
uri += '&dl=1'
# Read the RSS feed
msg = None
entries = None
if readout:
uri = uri.replace(' ', '%20')
logging.debug("Running feedparser on %s", uri)
d = feedparser.parse(uri.replace('feed://', 'http://'))
logging.debug("Done parsing %s", uri)
if not d:
msg = Ta('Failed to retrieve RSS from %s: %s') % (uri, '?')
logging.info(msg)
return unicoder(msg)
status = d.get('status', 999)
if status in (401, 402, 403):
msg = Ta('Do not have valid authentication for feed %s') % feed
logging.info(msg)
return unicoder(msg)
entries = d.get('entries')
if 'bozo_exception' in d and not entries:
msg = Ta('Failed to retrieve RSS from %s: %s') % (
uri, xml_name(str(d['bozo_exception'])))
logging.info(msg)
return unicoder(msg)
if not entries:
msg = Ta('RSS Feed %s was empty') % uri
logging.info(msg)
if feed not in self.jobs:
self.jobs[feed] = {}
jobs = self.jobs[feed]
if readout:
if not entries:
return unicoder(msg)
else:
entries = jobs.keys()
order = 0
# Filter out valid new links
for entry in entries:
if self.shutdown: return
if readout:
try:
link, category = _get_link(uri, entry)
except (AttributeError, IndexError):
link = None
category = ''
logging.info(Ta('Incompatible feed') + ' ' + uri)
logging.info("Traceback: ", exc_info=True)
return T('Incompatible feed')
category = latin1(category)
# Make sure only latin-1 encodable characters occur
atitle = latin1(entry.title)
title = unicoder(atitle)
else:
link = entry
category = jobs[link].get('orgcat', '')
if category in ('', '*'):
category = None
atitle = latin1(jobs[link].get('title', ''))
title = unicoder(atitle)
if link:
# Make sure spaces are quoted in the URL
if 'nzbclub.com' in link:
link, path = os.path.split(link.strip())
link = '%s/%s' % (link, urllib.quote(path))
else:
link = link.strip().replace(' ', '%20')
newlinks.append(link)
if link in jobs:
jobstat = jobs[link].get('status', ' ')[0]
else:
jobstat = 'N'
if jobstat in 'NGB' or (jobstat == 'X' and readout):
# Match this title against all filters
logging.debug('Trying title %s', atitle)
result = False
myCat = defCat
myPP = defPP
myScript = defScript
myPrio = defPrio
n = 0
# Match against all filters until an postive or negative match
for n in xrange(regcount):
if reEnabled[n]:
if category and reTypes[n] == 'C':
found = re.search(regexes[n], category)
if not found:
logging.debug("Filter rejected on rule %d",
n)
result = False
break
else:
if regexes[n]:
found = re.search(regexes[n], title)
else:
found = False
if reTypes[n] == 'M' and not found:
logging.debug("Filter rejected on rule %d",
n)
result = False
break
if found and reTypes[n] == 'A':
logging.debug("Filter matched on rule %d",
n)
result = True
break
if found and reTypes[n] == 'R':
logging.debug("Filter rejected on rule %d",
n)
result = False
break
if len(reCats):
if notdefault(reCats[n]):
myCat = reCats[n]
elif category and not defCat:
myCat = cat_convert(category)
if myCat:
myCat, catPP, catScript, catPrio = cat_to_opts(
myCat)
else:
myCat = catPP = catScript = catPrio = None
if notdefault(rePPs[n]):
myPP = rePPs[n]
elif not (reCats[n] or category):
myPP = catPP
if notdefault(reScripts[n]):
myScript = reScripts[n]
elif not (notdefault(reCats[n]) or category):
myScript = catScript
if rePrios[n] not in (str(DEFAULT_PRIORITY), ''):
myPrio = rePrios[n]
elif not ((rePrios[n] != str(DEFAULT_PRIORITY))
or category):
myPrio = catPrio
if cfg.no_dupes() and dup_title(title):
if cfg.no_dupes() == 1:
logging.info("Ignoring duplicate job %s", atitle)
continue
else:
myPrio = DUP_PRIORITY
act = download and not first
if link in jobs:
act = act and not jobs[link].get('status',
'').endswith('*')
act = act or force
star = first or jobs[link].get('status',
'').endswith('*')
else:
star = first
if result:
_HandleLink(jobs,
link,
title,
'G',
category,
myCat,
myPP,
myScript,
act,
star,
order,
priority=myPrio,
rule=str(n))
if act:
new_downloads.append(title)
else:
_HandleLink(jobs,
link,
title,
'B',
category,
myCat,
myPP,
myScript,
False,
star,
order,
priority=myPrio,
rule=str(n))
order += 1
# Send email if wanted and not "forced"
if new_downloads and cfg.email_rss() and not force:
emailer.rss_mail(feed, new_downloads)
remove_obsolete(jobs, newlinks)
return ''
def __init__(self):
def check_str(p):
return p is None or p == '' or isinstance(p, str)
def check_int(p):
try:
int(p)
return True
except:
return False
self.jobs = {}
try:
defined = config.get_rss().keys()
feeds = sabnzbd.load_admin(RSS_FILE_NAME)
if type(feeds) == type({}):
for feed in feeds:
if feed not in defined:
logging.debug('Dropping obsolete data for feed "%s"',
feed)
continue
self.jobs[feed] = {}
for link in feeds[feed]:
data = feeds[feed][link]
if type(data) == type([]):
# Convert previous list-based store to dictionary
new = {}
try:
new['status'] = data[0]
new['title'] = data[1]
new['url'] = data[2]
new['cat'] = data[3]
new['pp'] = data[4]
new['script'] = data[5]
new['time'] = data[6]
new['prio'] = str(NORMAL_PRIORITY)
new['rule'] = 0
self.jobs[feed][link] = new
except IndexError:
del new
else:
# Consistency check on data
try:
item = feeds[feed][link]
if not isinstance(
item, dict) or not isinstance(
item.get('title'), unicode):
raise IndexError
if item.get('status',
' ')[0] not in ('D', 'G', 'B',
'X'):
item['status'] = 'X'
if not isinstance(item.get('url'), unicode):
item['url'] = ''
item['url'] = item['url'].replace(
'www.newzbin.com', cfg.newzbin_url())
if not check_str(item.get('cat')):
item['cat'] = ''
if not check_str(item.get('orgcat')):
item['orgcat'] = ''
if not check_str(item.get('pp')):
item['pp'] = '3'
if not check_str(item.get('script')):
item['script'] = 'None'
if not check_str(item.get('prio')):
item['prio'] = '-100'
if not check_int(item.get('rule', 0)):
item['rule'] = 0
if not isinstance(item.get('time'), float):
item['time'] = time.time()
if not check_int(item.get('order', 0)):
item.get['order'] = 0
self.jobs[feed][link] = item
except (KeyError, IndexError):
logging.info(
'Incorrect entry in %s detected, discarding %s',
RSS_FILE_NAME, item)
remove_obsolete(self.jobs[feed], self.jobs[feed].keys())
except IOError:
logging.debug('Cannot read file %s', RSS_FILE_NAME)
# jobs is a NAME-indexed dictionary
# Each element is link-indexed dictionary
# Each element is another dictionary:
# status : 'D', 'G', 'B', 'X' (downloaded, good-match, bad-match, obsolete)
# '*' added means: from the initial batch
# '-' added to 'D' means downloaded, but not displayed anymore
# title : Title
# url : URL or MsgId
# cat : category
# orgcat : category as read from feed
# pp : pp
# script : script
# prio : priority
# time : timestamp (used for time-based clean-up)
# order : order in the RSS feed
self.shutdown = False