def process_job(nzo): """ Process one job """ start = time.time() # keep track of whether we can continue all_ok = True # keep track of par problems par_error = False # keep track of any unpacking errors unpack_error = False # Signal empty download, for when 'empty_postproc' is enabled empty = False nzb_list = [] # These need to be initialized in case of a crash workdir_complete = "" script_log = "" script_line = "" # Get the job flags nzo.save_attribs() flag_repair, flag_unpack, flag_delete = nzo.repair_opts # Normalize PP if flag_delete: flag_unpack = True if flag_unpack: flag_repair = True # Get the NZB name filename = nzo.final_name # Download-processes can mark job as failed if nzo.fail_msg: nzo.status = Status.FAILED nzo.save_attribs() all_ok = False par_error = True unpack_error = 1 try: # Get the folder containing the download result workdir = nzo.downpath tmp_workdir_complete = None # if no files are present (except __admin__), fail the job if all_ok and len(globber(workdir)) < 2: if nzo.precheck: _, ratio = nzo.check_availability_ratio() emsg = T( "Download might fail, only %s of required %s available" ) % (ratio, cfg.req_completion_rate()) else: emsg = T("Download failed - Not on your server(s)") empty = True emsg += " - https://sabnzbd.org/not-complete" nzo.fail_msg = emsg nzo.set_unpack_info("Fail", emsg) nzo.status = Status.FAILED # do not run unpacking or parity verification flag_repair = flag_unpack = False all_ok = cfg.empty_postproc() and empty if not all_ok: par_error = True unpack_error = 1 script = nzo.script logging.info( "Starting Post-Processing on %s => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s", filename, flag_repair, flag_unpack, flag_delete, script, nzo.cat, ) # Set complete dir to workdir in case we need to abort workdir_complete = workdir # Send post-processing notification notifier.send_notification(T("Post-processing"), nzo.final_name, "pp", nzo.cat) # Par processing, if enabled if all_ok and flag_repair: par_error, re_add = parring(nzo, workdir) if re_add: # Try to get more par files return False # If we don't need extra par2, we can disconnect if sabnzbd.nzbqueue.NzbQueue.do.actives( grabs=False) == 0 and cfg.autodisconnect(): # This was the last job, close server connections sabnzbd.downloader.Downloader.do.disconnect() # Sanitize the resulting files if sabnzbd.WIN32: sanitize_files_in_folder(workdir) # Check if user allows unsafe post-processing if flag_repair and cfg.safe_postproc(): all_ok = all_ok and not par_error if all_ok: # Fix encodings fix_unix_encoding(workdir) # Use dirs generated by direct-unpacker if nzo.direct_unpacker and nzo.direct_unpacker.unpack_dir_info: ( tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file, ) = nzo.direct_unpacker.unpack_dir_info else: # Generate extraction path tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = prepare_extraction_path( nzo) newfiles = [] # Run Stage 2: Unpack if flag_unpack: # Set the current nzo status to "Extracting...". Used in History nzo.status = Status.EXTRACTING logging.info("Running unpack_magic on %s", filename) unpack_error, newfiles = unpack_magic(nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), (), ()) logging.info("Unpacked files %s", newfiles) if sabnzbd.WIN32: # Sanitize the resulting files newfiles = sanitize_files_in_folder(tmp_workdir_complete) logging.info("Finished unpack_magic on %s", filename) if cfg.safe_postproc(): all_ok = all_ok and not unpack_error if all_ok: # Move any (left-over) files to destination nzo.status = Status.MOVING nzo.set_action_line(T("Moving"), "...") for root, _dirs, files in os.walk(workdir): if not root.endswith(JOB_ADMIN): for file_ in files: path = os.path.join(root, file_) new_path = path.replace(workdir, tmp_workdir_complete) ok, new_path = move_to_path(path, new_path) if new_path: newfiles.append(new_path) if not ok: nzo.set_unpack_info( "Unpack", T("Failed moving %s to %s") % (path, new_path)) all_ok = False break # Set permissions right set_permissions(tmp_workdir_complete) if all_ok and marker_file: del_marker(os.path.join(tmp_workdir_complete, marker_file)) remove_from_list(marker_file, newfiles) if all_ok: # Remove files matching the cleanup list cleanup_list(tmp_workdir_complete, skip_nzb=True) # Check if this is an NZB-only download, if so redirect to queue # except when PP was Download-only if flag_repair: nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, nzo.cat, nzo.priority) else: nzb_list = None if nzb_list: nzo.set_unpack_info("Download", T("Sent %s to queue") % nzb_list) cleanup_empty_directories(tmp_workdir_complete) else: # Full cleanup including nzb's cleanup_list(tmp_workdir_complete, skip_nzb=False) script_output = "" script_ret = 0 if not nzb_list: # Give destination its final name if cfg.folder_rename() and tmp_workdir_complete and not one_folder: if not all_ok: # Rename failed folders so they are easy to recognize workdir_complete = tmp_workdir_complete.replace( "_UNPACK_", "_FAILED_") workdir_complete = get_unique_path(workdir_complete, create_dir=False) try: newfiles = rename_and_collapse_folder( tmp_workdir_complete, workdir_complete, newfiles) except: logging.error( T('Error renaming "%s" to "%s"'), clip_path(tmp_workdir_complete), clip_path(workdir_complete), ) logging.info("Traceback: ", exc_info=True) # Better disable sorting because filenames are all off now file_sorter.sort_file = None if empty: job_result = -1 else: job_result = int(par_error) + int(bool(unpack_error)) * 2 if cfg.ignore_samples(): remove_samples(workdir_complete) # TV/Movie/Date Renaming code part 2 - rename and move files to parent folder if all_ok and file_sorter.sort_file: if newfiles: file_sorter.rename(newfiles, workdir_complete) workdir_complete, ok = file_sorter.move(workdir_complete) else: workdir_complete, ok = file_sorter.rename_with_ext( workdir_complete) if not ok: nzo.set_unpack_info("Unpack", T("Failed to move files")) all_ok = False if cfg.deobfuscate_final_filenames() and all_ok and not nzb_list: # deobfuscate the filenames logging.info("Running deobfuscate") deobfuscate.deobfuscate_list(newfiles, nzo.final_name) # Run the user script script_path = make_script_path(script) if (all_ok or not cfg.safe_postproc()) and ( not nzb_list) and script_path: # Set the current nzo status to "Ext Script...". Used in History nzo.status = Status.RUNNING nzo.set_action_line(T("Running script"), script) nzo.set_unpack_info("Script", T("Running user script %s") % script, unique=True) script_log, script_ret = external_processing( script_path, nzo, clip_path(workdir_complete), nzo.final_name, job_result) script_line = get_last_line(script_log) if script_log: script_output = nzo.nzo_id if script_line: nzo.set_unpack_info("Script", script_line, unique=True) else: nzo.set_unpack_info("Script", T("Ran %s") % script, unique=True) else: script = "" script_line = "" script_ret = 0 # Maybe bad script result should fail job if script_ret and cfg.script_can_fail(): script_error = True all_ok = False nzo.fail_msg = T("Script exit code is %s") % script_ret else: script_error = False # Email the results if (not nzb_list) and cfg.email_endjob(): if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error or script_error)): emailer.endjob( nzo.final_name, nzo.cat, all_ok, workdir_complete, nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info, script, script_log, script_ret, ) if script_output: # Can do this only now, otherwise it would show up in the email if script_ret: script_ret = "Exit(%s) " % script_ret else: script_ret = "" if len(script_log.rstrip().split("\n")) > 1: nzo.set_unpack_info( "Script", '%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, script_line, encoding.xml_name(script_output), T("More")), unique=True, ) else: # No '(more)' button needed nzo.set_unpack_info("Script", "%s%s " % (script_ret, script_line), unique=True) # Cleanup again, including NZB files if all_ok: cleanup_list(workdir_complete, False) # Force error for empty result all_ok = all_ok and not empty # Update indexer with results if cfg.rating_enable(): if nzo.encrypted > 0: Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED) if empty: hosts = [ s.host for s in sabnzbd.downloader.Downloader.do.nzo_servers(nzo) ] if not hosts: hosts = [None] for host in hosts: Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host) except: logging.error(T("Post Processing Failed for %s (%s)"), filename, T("see logfile")) logging.info("Traceback: ", exc_info=True) nzo.fail_msg = T("PostProcessing was aborted (%s)") % T("see logfile") notifier.send_notification(T("Download Failed"), filename, "failed", nzo.cat) nzo.status = Status.FAILED par_error = True all_ok = False if cfg.email_endjob(): emailer.endjob( nzo.final_name, nzo.cat, all_ok, clip_path(workdir_complete), nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info, "", "", 0, ) if all_ok: # If the folder only contains one file OR folder, have that as the path # Be aware that series/generic/date sorting may move a single file into a folder containing other files workdir_complete = one_file_or_folder(workdir_complete) workdir_complete = os.path.normpath(workdir_complete) # Clean up the NZO data try: nzo.purge_data(delete_all_data=all_ok) except: logging.error(T("Cleanup of %s failed."), nzo.final_name) logging.info("Traceback: ", exc_info=True) # Use automatic retry link on par2 errors and encrypted/bad RARs if par_error or unpack_error in (2, 3): try_alt_nzb(nzo) # Show final status in history if all_ok: notifier.send_notification(T("Download Completed"), filename, "complete", nzo.cat) nzo.status = Status.COMPLETED else: notifier.send_notification(T("Download Failed"), filename, "failed", nzo.cat) nzo.status = Status.FAILED # Log the overall time taken for postprocessing postproc_time = int(time.time() - start) # Create the history DB instance history_db = database.HistoryDB() # Add the nzo to the database. Only the path, script and time taken is passed # Other information is obtained from the nzo history_db.add_history_db(nzo, workdir_complete, postproc_time, script_log, script_line) # Purge items history_db.auto_history_purge() # The connection is only used once, so close it here history_db.close() sabnzbd.history_updated() return True
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False def dup_title(title): """ Check if this title was in this or other feeds Return matching feed name """ title = title.lower() for fd in self.jobs: for lk in self.jobs[fd]: item = self.jobs[fd][lk] if item.get('status', ' ')[0] == 'D' and \ item.get('title', '').lower() == title: return fd return '' if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(T('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info=True) return T('Incorrect RSS feed description "%s"') % feed uri = feeds.uri() defCat = feeds.cat() import sabnzbd.api if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) if filter[3] in ('<', '>', 'F'): regexes.append(filter[4]) else: regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Check for nzbs.org if 'nzbs.org/' in uri and '&dl=1' not in uri: uri += '&dl=1' # Read the RSS feed msg = None entries = None if readout: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) d = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not d: msg = T('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) return unicoder(msg) status = d.get('status', 999) if status in (401, 402, 403): msg = T('Do not have valid authentication for feed %s') % feed logging.info(msg) return unicoder(msg) if status >= 500 and status <= 599: msg = T('Server side error (server code %s); could not get %s on %s') % (status, feed, uri) logging.info(msg) return unicoder(msg) entries = d.get('entries') if 'bozo_exception' in d and not entries: msg = str(d['bozo_exception']) if 'CERTIFICATE_VERIFY_FAILED' in msg: msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri) logging.error(msg) else: msg = T('Failed to retrieve RSS from %s: %s') % (uri, xml_name(msg)) logging.info(msg) return unicoder(msg) if not entries: msg = T('RSS Feed %s was empty') % uri logging.info(msg) if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() # Sort in the order the jobs came from the feed entries.sort(lambda x, y: jobs[x].get('order', 0) - jobs[y].get('order', 0)) order = 0 # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category, size = _get_link(uri, entry) except (AttributeError, IndexError): link = None category = u'' size = 0L logging.info(T('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info=True) return T('Incompatible feed') title = entry.title else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None title = jobs[link].get('title', '') size = jobs[link].get('size', 0L) if link: # Make sure spaces are quoted in the URL link = link.strip().replace(' ', '%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', title) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 if 'F' in reTypes: season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3] season = int_conv(season) episode = int_conv(episode) else: season = episode = 0 # Match against all filters until an positive or negative match logging.debug('Size %s for %s', size, title) for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break elif reTypes[n] == '<' and size and from_units(regexes[n]) < size: # "Size at most" : too large logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == '>' and size and from_units(regexes[n]) > size: # "Size at least" : too small logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == 'F' and not ep_match(season, episode, regexes[n]): # "Starting from SxxEyy", too early episode logging.debug('Filter requirement match on rule %d', n) result = False break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if notdefault(reCats[n]): myCat = reCats[n] elif category and not defCat: myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts(myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and dup_title(title): if cfg.no_dupes() == 1: logging.info("Ignoring duplicate job %s", title) continue else: myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, link, title, size, 'G', category, myCat, myPP, myScript, act, star, order, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, link, title, size, 'B', category, myCat, myPP, myScript, False, star, order, priority=myPrio, rule=str(n)) order += 1 # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return ''
try: nw = NewsWrapper(s, -1, block=True) nw.init_connect(None) while not nw.connected: nw.lines = [] nw.recv_chunk(block=True) nw.finish_connect(nw.lines[0][:3]) except socket.timeout, e: if port != 119 and not ssl: return False, T('Timed out: Try enabling SSL or connecting on a different port.') else: return False, T('Timed out') except socket.error, e: return False, xml_name(str(e)) except TypeError, e: return False, xml_name(T('Invalid server address.')) except IndexError: # No data was received in recv_chunk() call return False, xml_name(T('Server quit during login sequence.')) except: return False, xml_name(str(sys.exc_info()[1])) if not username or not password: nw.nntp.sock.sendall('ARTICLE test\r\n') try:
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(T('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info=True) return T('Incorrect RSS feed description "%s"') % feed uris = feeds.uri() defCat = feeds.cat() import sabnzbd.api if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) if filter[3] in ('<', '>', 'F', 'S'): regexes.append(filter[4]) else: regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Read the RSS feed msg = None entries = None if readout: all_entries = [] for uri in uris: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) feed_parsed = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not feed_parsed: msg = T('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) status = feed_parsed.get('status', 999) if status in (401, 402, 403): msg = T('Do not have valid authentication for feed %s') % feed logging.info(msg) if status >= 500 and status <= 599: msg = T('Server side error (server code %s); could not get %s on %s') % (status, feed, uri) logging.info(msg) entries = feed_parsed.get('entries') if 'bozo_exception' in feed_parsed and not entries: msg = str(feed_parsed['bozo_exception']) if 'CERTIFICATE_VERIFY_FAILED' in msg: msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri) msg += ' - https://sabnzbd.org/certificate-errors' logging.error(msg) else: msg = T('Failed to retrieve RSS from %s: %s') % (uri, xml_name(msg)) logging.info(msg) if not entries: msg = T('RSS Feed %s was empty') % uri logging.info(msg) all_entries.extend(entries) entries = all_entries if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() # Sort in the order the jobs came from the feed entries.sort(lambda x, y: jobs[x].get('order', 0) - jobs[y].get('order', 0)) order = 0 # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category, size, age, season, episode = _get_link(uri, entry) except (AttributeError, IndexError): link = None category = u'' size = 0L age = None logging.info(T('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info=True) return T('Incompatible feed') title = entry.title # If there's multiple feeds, remove the duplicates based on title and size if len(uris) > 1: skip_job = False for job_link, job in jobs.items(): # Allow 5% size deviation because indexers might have small differences for same release if job.get('title') == title and link != job_link and (job.get('size')*0.95) < size < (job.get('size')*1.05): logging.info("Ignoring job %s from other feed", title) skip_job = True break if skip_job: continue else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None title = jobs[link].get('title', '') size = jobs[link].get('size', 0L) age = jobs[link].get('age') season = jobs[link].get('season', 0) episode = jobs[link].get('episode', 0) if link: # Make sure spaces are quoted in the URL link = link.strip().replace(' ', '%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', title) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 if ('F' in reTypes or 'S' in reTypes) and (not season or not episode): season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3] season = int_conv(season) episode = int_conv(episode) # Match against all filters until an positive or negative match logging.debug('Size %s for %s', size, title) for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break elif reTypes[n] == '<' and size and from_units(regexes[n]) < size: # "Size at most" : too large logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == '>' and size and from_units(regexes[n]) > size: # "Size at least" : too small logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == 'F' and not ep_match(season, episode, regexes[n]): # "Starting from SxxEyy", too early episode logging.debug('Filter requirement match on rule %d', n) result = False break elif reTypes[n] == 'S' and season and episode and ep_match(season, episode, regexes[n], title): logging.debug('Filter matched on rule %d', n) result = True break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if not result and defCat: # Apply Feed-category on non-matched items myCat = defCat elif result and notdefault(reCats[n]): # Use the matched info myCat = reCats[n] elif category and not defCat: # No result and no Feed-category myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts(myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and self.check_duplicate(title): if cfg.no_dupes() == 1: # Dupe-detection: Discard logging.info("Ignoring duplicate job %s", title) continue elif cfg.no_dupes() == 3: # Dupe-detection: Fail # We accept it so the Queue can send it to the History logging.info("Found duplicate job %s", title) else: # Dupe-detection: Pause myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, link, title, size, age, season, episode, 'G', category, myCat, myPP, myScript, act, star, order, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, link, title, size, age, season, episode, 'B', category, myCat, myPP, myScript, False, star, order, priority=myPrio, rule=str(n)) order += 1 # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return msg
nw = NewsWrapper(s, -1, block=True) nw.init_connect(None) while not nw.connected: nw.lines = [] nw.recv_chunk(block=True) nw.finish_connect(nw.lines[0][:3]) except socket.timeout, e: if port != 119 and not ssl: return False, T( 'Timed out: Try enabling SSL or connecting on a different port.' ) else: return False, T('Timed out') except socket.error, e: return False, xml_name(str(e)) except TypeError, e: return False, xml_name(T('Invalid server address.')) except: return False, xml_name(str(sys.exc_info()[1])) if not username or not password: nw.nntp.sock.sendall('ARTICLE test\r\n') try: nw.lines = [] nw.recv_chunk(block=True) except: return False, xml_name(str(sys.exc_info()[1]))
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False def dup_title(title): """ Check if this title was in this or other feeds Return matching feed name """ title = title.lower() for fd in self.jobs: for lk in self.jobs[fd]: item = self.jobs[fd][lk] if item.get('status', ' ')[0] == 'D' and \ item.get('title', '').lower() == title: return fd return '' if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(Ta('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info = True) return T('Incorrect RSS feed description "%s"') % feed uri = feeds.uri() defCat = feeds.cat() if not notdefault(defCat): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Check for nzbs.org if 'nzbs.org/' in uri and not ('&dl=1' in uri): uri += '&dl=1' # Read the RSS feed msg = None entries = None if readout: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) d = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not d: msg = Ta('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) return unicoder(msg) status = d.get('status', 999) if status in (401, 402, 403): msg = Ta('Do not have valid authentication for feed %s') % feed logging.info(msg) return unicoder(msg) entries = d.get('entries') if 'bozo_exception' in d and not entries: msg = Ta('Failed to retrieve RSS from %s: %s') % (uri, xml_name(str(d['bozo_exception']))) logging.info(msg) return unicoder(msg) if not entries: msg = Ta('RSS Feed %s was empty') % uri logging.info(msg) if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() order = 0 # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category = _get_link(uri, entry) except (AttributeError, IndexError): link = None category = '' logging.info(Ta('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info = True) return T('Incompatible feed') category = latin1(category) # Make sure only latin-1 encodable characters occur atitle = latin1(entry.title) title = unicoder(atitle) else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None atitle = latin1(jobs[link].get('title', '')) title = unicoder(atitle) if link: # Make sure spaces are quoted in the URL if 'nzbclub.com' in link: link, path = os.path.split(link.strip()) link = '%s/%s' % (link, urllib.quote(path)) else: link = link.strip().replace(' ','%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', atitle) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 # Match against all filters until an postive or negative match for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if notdefault(reCats[n]): myCat = reCats[n] elif category and not defCat: myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts(myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and dup_title(title): if cfg.no_dupes() == 1: logging.info("Ignoring duplicate job %s", atitle) continue else: myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, link, title, 'G', category, myCat, myPP, myScript, act, star, order, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, link, title, 'B', category, myCat, myPP, myScript, False, star, order, priority=myPrio, rule=str(n)) order += 1 # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return ''
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False def dup_title(title): """ Check if this title was in this or other feeds Return matching feed name """ title = title.lower() for fd in self.jobs: for lk in self.jobs[fd]: item = self.jobs[fd][lk] if item.get('status', ' ')[0] == 'D' and \ item.get('title', '').lower() == title: return fd return '' if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(Ta('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info=True) return T('Incorrect RSS feed description "%s"') % feed uri = feeds.uri() defCat = feeds.cat() if not notdefault(defCat): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Check for nzbs.org if 'nzbs.org/' in uri and not ('&dl=1' in uri): uri += '&dl=1' # Read the RSS feed msg = None entries = None if readout: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) d = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not d: msg = Ta('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) return unicoder(msg) status = d.get('status', 999) if status in (401, 402, 403): msg = Ta('Do not have valid authentication for feed %s') % feed logging.info(msg) return unicoder(msg) entries = d.get('entries') if 'bozo_exception' in d and not entries: msg = Ta('Failed to retrieve RSS from %s: %s') % ( uri, xml_name(str(d['bozo_exception']))) logging.info(msg) return unicoder(msg) if not entries: msg = Ta('RSS Feed %s was empty') % uri logging.info(msg) if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() order = 0 # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category = _get_link(uri, entry) except (AttributeError, IndexError): link = None category = '' logging.info(Ta('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info=True) return T('Incompatible feed') category = latin1(category) # Make sure only latin-1 encodable characters occur atitle = latin1(entry.title) title = unicoder(atitle) else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None atitle = latin1(jobs[link].get('title', '')) title = unicoder(atitle) if link: # Make sure spaces are quoted in the URL if 'nzbclub.com' in link: link, path = os.path.split(link.strip()) link = '%s/%s' % (link, urllib.quote(path)) else: link = link.strip().replace(' ', '%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', atitle) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 # Match against all filters until an postive or negative match for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if notdefault(reCats[n]): myCat = reCats[n] elif category and not defCat: myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts( myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and dup_title(title): if cfg.no_dupes() == 1: logging.info("Ignoring duplicate job %s", atitle) continue else: myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, link, title, 'G', category, myCat, myPP, myScript, act, star, order, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, link, title, 'B', category, myCat, myPP, myScript, False, star, order, priority=myPrio, rule=str(n)) order += 1 # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return ''
def run_feed(self, feed=None, download=False, ignoreFirst=False, force=False, readout=True): """ Run the query for one URI and apply filters """ self.shutdown = False if not feed: return 'No such feed' newlinks = [] new_downloads = [] # Preparations, get options try: feeds = config.get_rss()[feed] except KeyError: logging.error(T('Incorrect RSS feed description "%s"'), feed) logging.info("Traceback: ", exc_info=True) return T('Incorrect RSS feed description "%s"') % feed uris = feeds.uri() defCat = feeds.cat() import sabnzbd.api if not notdefault(defCat) or defCat not in sabnzbd.api.list_cats(default=False): defCat = None defPP = feeds.pp() if not notdefault(defPP): defPP = None defScript = feeds.script() if not notdefault(defScript): defScript = None defPrio = feeds.priority() if not notdefault(defPrio): defPrio = None # Preparations, convert filters to regex's regexes = [] reTypes = [] reCats = [] rePPs = [] rePrios = [] reScripts = [] reEnabled = [] for filter in feeds.filters(): reCat = filter[0] if defCat in ('', '*'): reCat = None reCats.append(reCat) rePPs.append(filter[1]) reScripts.append(filter[2]) reTypes.append(filter[3]) if filter[3] in ('<', '>', 'F', 'S'): regexes.append(filter[4]) else: regexes.append(convert_filter(filter[4])) rePrios.append(filter[5]) reEnabled.append(filter[6] != '0') regcount = len(regexes) # Set first if this is the very first scan of this URI first = (feed not in self.jobs) and ignoreFirst # Add sabnzbd's custom User Agent feedparser.USER_AGENT = 'SABnzbd+/%s' % sabnzbd.version.__version__ # Read the RSS feed msg = None entries = None if readout: all_entries = [] for uri in uris: uri = uri.replace(' ', '%20') logging.debug("Running feedparser on %s", uri) feed_parsed = feedparser.parse(uri.replace('feed://', 'http://')) logging.debug("Done parsing %s", uri) if not feed_parsed: msg = T('Failed to retrieve RSS from %s: %s') % (uri, '?') logging.info(msg) status = feed_parsed.get('status', 999) if status in (401, 402, 403): msg = T('Do not have valid authentication for feed %s') % feed logging.info(msg) if 500 <= status <= 599: msg = T('Server side error (server code %s); could not get %s on %s') % (status, feed, uri) logging.info(msg) entries = feed_parsed.get('entries') if 'bozo_exception' in feed_parsed and not entries: msg = str(feed_parsed['bozo_exception']) if 'CERTIFICATE_VERIFY_FAILED' in msg: msg = T('Server %s uses an untrusted HTTPS certificate') % get_urlbase(uri) msg += ' - https://sabnzbd.org/certificate-errors' logging.error(msg) else: msg = T('Failed to retrieve RSS from %s: %s') % (uri, xml_name(msg)) logging.info(msg) if not entries: msg = T('RSS Feed %s was empty') % uri logging.info(msg) all_entries.extend(entries) entries = all_entries # In case of a new feed if feed not in self.jobs: self.jobs[feed] = {} jobs = self.jobs[feed] # Error in readout or now new readout if readout: if not entries: return unicoder(msg) else: entries = jobs.keys() # Filter out valid new links for entry in entries: if self.shutdown: return if readout: try: link, category, size, age, season, episode = _get_link(entry) except (AttributeError, IndexError): logging.info(T('Incompatible feed') + ' ' + uri) logging.info("Traceback: ", exc_info=True) return T('Incompatible feed') title = entry.title # If there's multiple feeds, remove the duplicates based on title and size if len(uris) > 1: skip_job = False for job_link, job in jobs.items(): # Allow 5% size deviation because indexers might have small differences for same release if job.get('title') == title and link != job_link and (job.get('size')*0.95) < size < (job.get('size')*1.05): logging.info("Ignoring job %s from other feed", title) skip_job = True break if skip_job: continue else: link = entry category = jobs[link].get('orgcat', '') if category in ('', '*'): category = None title = jobs[link].get('title', '') size = jobs[link].get('size', 0L) age = jobs[link].get('age') season = jobs[link].get('season', 0) episode = jobs[link].get('episode', 0) if link: # Make sure spaces are quoted in the URL link = link.strip().replace(' ', '%20') newlinks.append(link) if link in jobs: jobstat = jobs[link].get('status', ' ')[0] else: jobstat = 'N' if jobstat in 'NGB' or (jobstat == 'X' and readout): # Match this title against all filters logging.debug('Trying title %s', title) result = False myCat = defCat myPP = defPP myScript = defScript myPrio = defPrio n = 0 if ('F' in reTypes or 'S' in reTypes) and (not season or not episode): season, episode = sabnzbd.newsunpack.analyse_show(title)[1:3] # Match against all filters until an positive or negative match logging.debug('Size %s', size) for n in xrange(regcount): if reEnabled[n]: if category and reTypes[n] == 'C': found = re.search(regexes[n], category) if not found: logging.debug("Filter rejected on rule %d", n) result = False break elif reTypes[n] == '<' and size and from_units(regexes[n]) < size: # "Size at most" : too large logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == '>' and size and from_units(regexes[n]) > size: # "Size at least" : too small logging.debug('Filter rejected on rule %d', n) result = False break elif reTypes[n] == 'F' and not ep_match(season, episode, regexes[n]): # "Starting from SxxEyy", too early episode logging.debug('Filter requirement match on rule %d', n) result = False break elif reTypes[n] == 'S' and season and episode and ep_match(season, episode, regexes[n], title): logging.debug('Filter matched on rule %d', n) result = True break else: if regexes[n]: found = re.search(regexes[n], title) else: found = False if reTypes[n] == 'M' and not found: logging.debug("Filter rejected on rule %d", n) result = False break if found and reTypes[n] == 'A': logging.debug("Filter matched on rule %d", n) result = True break if found and reTypes[n] == 'R': logging.debug("Filter rejected on rule %d", n) result = False break if len(reCats): if not result and defCat: # Apply Feed-category on non-matched items myCat = defCat elif result and notdefault(reCats[n]): # Use the matched info myCat = reCats[n] elif category and not defCat: # No result and no Feed-category myCat = cat_convert(category) if myCat: myCat, catPP, catScript, catPrio = cat_to_opts(myCat) else: myCat = catPP = catScript = catPrio = None if notdefault(rePPs[n]): myPP = rePPs[n] elif not (reCats[n] or category): myPP = catPP if notdefault(reScripts[n]): myScript = reScripts[n] elif not (notdefault(reCats[n]) or category): myScript = catScript if rePrios[n] not in (str(DEFAULT_PRIORITY), ''): myPrio = rePrios[n] elif not ((rePrios[n] != str(DEFAULT_PRIORITY)) or category): myPrio = catPrio if cfg.no_dupes() and self.check_duplicate(title): if cfg.no_dupes() == 1: # Dupe-detection: Discard logging.info("Ignoring duplicate job %s", title) continue elif cfg.no_dupes() == 3: # Dupe-detection: Fail # We accept it so the Queue can send it to the History logging.info("Found duplicate job %s", title) else: # Dupe-detection: Pause myPrio = DUP_PRIORITY act = download and not first if link in jobs: act = act and not jobs[link].get('status', '').endswith('*') act = act or force star = first or jobs[link].get('status', '').endswith('*') else: star = first if result: _HandleLink(jobs, feed, link, title, size, age, season, episode, 'G', category, myCat, myPP, myScript, act, star, priority=myPrio, rule=str(n)) if act: new_downloads.append(title) else: _HandleLink(jobs, feed, link, title, size, age, season, episode, 'B', category, myCat, myPP, myScript, False, star, priority=myPrio, rule=str(n)) # Send email if wanted and not "forced" if new_downloads and cfg.email_rss() and not force: emailer.rss_mail(feed, new_downloads) remove_obsolete(jobs, newlinks) return msg
nw = NewsWrapper(s, -1, block=True) nw.init_connect(None) while not nw.connected: nw.lines = [] nw.recv_chunk(block=True) nw.finish_connect(nw.lines[0][:3]) except socket.timeout, e: if port != 119 and not ssl: return False, T( 'Timed out: Try enabling SSL or connecting on a different port.' ) else: return False, T('Timed out') except socket.error, e: return False, xml_name(str(e)) except TypeError, e: return False, xml_name(T('Invalid server address.')) except IndexError: # No data was received in recv_chunk() call return False, xml_name(T('Server quit during login sequence.')) except: return False, xml_name(str(sys.exc_info()[1])) if not username or not password: nw.nntp.sock.sendall('ARTICLE <test@home>\r\n') try: nw.lines = []