def _analyse_others(fn, url): """ Analyse respons of indexer returns fn|None, error-message|None, retry, wait-seconds """ msg = '' wait = 0 if not fn: logging.debug('No response from indexer, retry after 60 sec') return None, msg, True, 60 try: f = open(fn, 'r') data = f.read(100) f.close() except: logging.debug('Problem with tempfile %s from indexer, retry after 60 sec', fn) return None, msg, True, 60 # Check for an error response if not data: logging.debug('Received nothing from indexer, retry after 60 sec') return None, msg, True, 60 if '.nzbsrus.' in url: # Partial support for nzbsrus.com's API if misc.match_str(data, RUS_FATAL): logging.debug('nzbsrus says: %s, abort', data) return None, data, False, 0 if misc.match_str(data, RUS_15M): logging.debug('nzbsrus says: %s, wait 15m', data) return None, data, True, 900 if misc.match_str(data, RUS_60M): logging.debug('nzbsrus says: %s, wait 60m', data) return None, data, True, 3600 return fn, msg, False, 0
def _analyse_matrix(fn, matrix_id): """ Analyse respons of nzbmatrix returns fn|None, error-message|None, retry, wait-seconds """ msg = '' wait = 0 if not fn: logging.debug('No response from nzbmatrix, retry after 60 sec') return None, msg, True, 60 try: f = open(fn, 'r') data = f.read(40).lower() f.close() except: logging.debug( 'Problem with tempfile %s from nzbmatrix, retry after 60 sec', fn) return None, msg, True, 60 # Check for an error response if data and '<!DOCTYPE' in data: # We got HTML, probably a temporary problem, keep trying msg = Ta('Invalid nzbmatrix report number %s') % matrix_id wait = 300 elif data and data.startswith('error'): txt = misc.match_str( data, ('invalid_login', 'invalid_api', 'disabled_account', 'vip_only')) if txt: if 'vip' in txt: msg = Ta('You need an nzbmatrix VIP account to use the API') else: msg = (Ta('Invalid nzbmatrix credentials') + ' (%s)') % txt return None, msg, False, 0 elif 'limit_reached' in data: msg = 'Too many nzbmatrix hits, waiting 10 min' wait = 600 elif misc.match_str(data, ('daily_limit', 'limit is reached')): # Daily limit reached, just wait an hour before trying again msg = 'Daily limit nzbmatrix reached, waiting 1 hour' wait = 3600 elif 'no_nzb_found' in data: msg = Ta('Invalid nzbmatrix report number %s') % matrix_id wait = 300 else: # Check if we are required to wait - if so sleep the urlgrabber m = _RE_MATRIX_ERR.search(data) if m: wait = min(int(m.group(1)), 600) else: msg = Ta('Problem accessing nzbmatrix server (%s)') % data wait = 60 if wait: # Return, but tell the urlgrabber to retry return None, msg, True, wait return fn, msg, False, 0
def _analyse_matrix(fn, matrix_id): """ Analyse respons of nzbmatrix returns fn|None, error-message|None, retry, wait-seconds """ msg = '' wait = 0 if not fn: logging.debug('No response from nzbmatrix, retry after 60 sec') return None, msg, True, 60 try: f = open(fn, 'r') data = f.read(40).lower() f.close() except: logging.debug('Problem with tempfile %s from nzbmatrix, retry after 60 sec', fn) return None, msg, True, 60 # Check for an error response if data and '<!DOCTYPE' in data: # We got HTML, probably a temporary problem, keep trying msg = Ta('Invalid nzbmatrix report number %s') % matrix_id wait = 300 elif data and data.startswith('error'): txt = misc.match_str(data, ('invalid_login', 'invalid_api', 'disabled_account', 'vip_only')) if txt: if 'vip' in txt: msg = Ta('You need an nzbmatrix VIP account to use the API') else: msg = (Ta('Invalid nzbmatrix credentials') + ' (%s)') % txt return None, msg, False, 0 elif 'limit_reached' in data: msg = 'Too many nzbmatrix hits, waiting 10 min' wait = 600 elif misc.match_str(data, ('daily_limit', 'limit is reached')): # Daily limit reached, just wait an hour before trying again msg = 'Daily limit nzbmatrix reached, waiting 1 hour' wait = 3600 elif 'no_nzb_found' in data: msg = Ta('Invalid nzbmatrix report number %s') % matrix_id wait = 300 else: # Check if we are required to wait - if so sleep the urlgrabber m = _RE_MATRIX_ERR.search(data) if m: wait = min(int(m.group(1)), 600) else: msg = Ta('Problem accessing nzbmatrix server (%s)') % data wait = 60 if wait: # Return, but tell the urlgrabber to retry return None, msg, True, wait return fn, msg, False, 0
def set_bonjour(host=None, port=None): """ Publish host/port combo through Bonjour """ global _HOST_PORT, _BONJOUR_OBJECT if not _HAVE_BONJOUR or not cfg.enable_bonjour(): logging.info('No Bonjour/ZeroConfig support installed') return if host is None and port is None: host, port = _HOST_PORT else: _HOST_PORT = (host, port) scope = pybonjour.kDNSServiceInterfaceIndexAny zhost = None domain = None if match_str(host, ('localhost', '127.0.', '::1')): logging.info('Bonjour/ZeroConfig does not support "localhost"') # All implementations fail to implement "localhost" properly # A false address is published even when scope==kDNSServiceInterfaceIndexLocalOnly return name = hostname() if '.local' in name: suffix = '' else: suffix = '.local' logging.debug('Try to publish in Bonjour as "%s" (%s:%s)', name, host, port) try: refObject = pybonjour.DNSServiceRegister( interfaceIndex=scope, name='SABnzbd on %s:%s' % (name, port), regtype='_http._tcp', domain=domain, host=zhost, port=int(port), txtRecord=pybonjour.TXTRecord({ 'path': cfg.url_base(), 'https': cfg.enable_https() }), callBack=_zeroconf_callback) except sabnzbd.utils.pybonjour.BonjourError as e: _BONJOUR_OBJECT = None logging.debug('Failed to start Bonjour service: %s', str(e)) except: _BONJOUR_OBJECT = None logging.debug( 'Failed to start Bonjour service due to non-pybonjour related problem', exc_info=True) else: Thread(target=_bonjour_server, args=(refObject, )) _BONJOUR_OBJECT = refObject logging.debug('Successfully started Bonjour service')
def _analyse_others(fn, url): """ Analyse respons of indexer returns fn|None, error-message|None, retry, wait-seconds """ msg = '' wait = 0 if not fn: logging.debug('No response from indexer, retry after 60 sec') return None, msg, True, 60 try: f = open(fn, 'r') data = f.read(100) f.close() except: logging.debug( 'Problem with tempfile %s from indexer, retry after 60 sec', fn) return None, msg, True, 60 # Check for an error response if not data: logging.debug('Received nothing from indexer, retry after 60 sec') return None, msg, True, 60 if '.nzbsrus.' in url: # Partial support for nzbsrus.com's API if misc.match_str(data, RUS_FATAL): logging.debug('nzbsrus says: %s, abort', data) return None, data, False, 0 if misc.match_str(data, RUS_15M): logging.debug('nzbsrus says: %s, wait 15m', data) return None, data, True, 900 if misc.match_str(data, RUS_60M): logging.debug('nzbsrus says: %s, wait 60m', data) return None, data, True, 3600 return fn, msg, False, 0
def _analyse_others(fn, url): """ Analyse respons of indexer returns fn|None, error-message|None, retry, wait-seconds """ msg = '' wait = 0 if not fn: logging.debug('No response from indexer, retry after 60 sec') return None, msg, True, 60 try: f = open(fn, 'r') data = f.read(100) f.close() except: logging.debug('Problem with tempfile %s from indexer, retry after 60 sec', fn) return None, msg, True, 60 # Check for an error response if not data: logging.debug('Received nothing from indexer, retry after 60 sec') return None, msg, True, 60 if '.nzbsrus.' in url: # Provisional support for nzbsrus.com's lack of an API # Trying to make sense of their response # Their non-VIP limiting is particularly weak f = open(fn, 'r') data = f.read(10000) f.close() ldata = data[:100].lower() if misc.match_str(ldata, ('invalid link', 'nuked', 'deleted')): logging.debug('nzbsrus says: %s, abort', data) return None, data, False, 0 if 'temporarily' in ldata: logging.debug('nzbsrus says: %s, retry', data) return None, data, True, 600 if 'Upgrade To ViP' in data: logging.debug('nzbsrus says: upgrade to VIP, retry after an hour') return None, 'upgrade to VIP', True, 3600 if 'Maintenance' in data: logging.debug('nzbsrus says: Maintenance, retry after an hour') return None, 'Maintenance', True, 3600 if '<nzb' not in ldata and '<!doctype' in ldata: msg = Ta('Invalid URL for nzbsrus') logging.debug(msg) return None, msg, False, 0 return fn, msg, False, 0
def set_bonjour(host=None, port=None): """ Publish host/port combo through Bonjour """ global _HOST_PORT, _BONJOUR_OBJECT if not _HAVE_BONJOUR or not cfg.enable_bonjour(): logging.info('No Bonjour/ZeroConfig support installed') return if host is None and port is None: host, port = _HOST_PORT else: _HOST_PORT = (host, port) scope = pybonjour.kDNSServiceInterfaceIndexAny zhost = None domain = None if match_str(host, ('localhost', '127.0.', '::1')): logging.info('Bonjour/ZeroConfig does not support "localhost"') # All implementations fail to implement "localhost" properly # A false address is published even when scope==kDNSServiceInterfaceIndexLocalOnly return name = hostname() if '.local' in name: suffix = '' else: suffix = '.local' if hasattr(cherrypy.wsgiserver, 'redirect_url'): cherrypy.wsgiserver.redirect_url("https://%s%s:%s/sabnzbd" % (name, suffix, port)) logging.debug('Try to publish in Bonjour as "%s" (%s:%s)', name, host, port) try: refObject = pybonjour.DNSServiceRegister( interfaceIndex=scope, name='SABnzbd on %s:%s' % (name, port), regtype='_http._tcp', domain=domain, host=zhost, port=int(port), txtRecord=pybonjour.TXTRecord({'path': '/sabnzbd/'}), callBack=_zeroconf_callback) except sabnzbd.utils.pybonjour.BonjourError: _BONJOUR_OBJECT = None logging.debug('Failed to start Bonjour service') else: Thread(target=_bonjour_server, args=(refObject,)) _BONJOUR_OBJECT = refObject logging.debug('Successfully started Bonjour service')
def special_rss_site(url): """ Return True if url describes an RSS site with odd titles """ return cfg.rss_filenames() or match_str(url, cfg.rss_odd_titles())
def run(self): while 1: # Sleep to allow decoder/assembler switching sleep(0.0001) art_tup = self.queue.get() if not art_tup: break article, lines, raw_data = art_tup nzf = article.nzf nzo = nzf.nzo art_id = article.article killed = False # Check if the space that's now free can let us continue the queue? qsize = self.queue.qsize() if (sabnzbd.articlecache.ArticleCache.do.free_reserve_space(lines) or qsize < MAX_DECODE_QUEUE) and \ (qsize < LIMIT_DECODE_QUEUE) and sabnzbd.downloader.Downloader.do.delayed: sabnzbd.downloader.Downloader.do.undelay() data = None register = True # Finish article found = False # Proper article found logme = None if lines or raw_data: try: if nzo.precheck: raise BadYenc register = True if self.__log_decoding: logging.debug("Decoding %s", art_id) data = self.decode(article, lines, raw_data) nzf.article_count += 1 found = True except IOError: logme = T('Decoding %s failed') % art_id logging.warning(logme) logging.info("Traceback: ", exc_info=True) sabnzbd.downloader.Downloader.do.pause() sabnzbd.nzbqueue.NzbQueue.do.reset_try_lists(article) register = False except MemoryError: logme = T('Decoder failure: Out of memory') logging.warning(logme) anfo = sabnzbd.articlecache.ArticleCache.do.cache_info() logging.info("Decoder-Queue: %d, Cache: %d, %d, %d", self.queue.qsize(), anfo.article_sum, anfo.cache_size, anfo.cache_limit) logging.info("Traceback: ", exc_info=True) sabnzbd.downloader.Downloader.do.pause() sabnzbd.nzbqueue.NzbQueue.do.reset_try_lists(article) register = False except CrcError, e: logme = 'CRC Error in %s' % art_id logging.info(logme) data = e.data except (BadYenc, ValueError): # Handles precheck and badly formed articles killed = False found = False data_to_check = lines or raw_data if nzo.precheck and data_to_check and data_to_check[ 0].startswith('223 '): # STAT was used, so we only get a status code found = True else: # Examine headers (for precheck) or body (for download) # And look for DMCA clues (while skipping "X-" headers) for line in data_to_check: lline = line.lower() if 'message-id:' in lline: found = True if not line.startswith('X-') and match_str( lline, ('dmca', 'removed', 'cancel', 'blocked')): killed = True break if killed: logme = 'Article removed from server (%s)' logging.info(logme, art_id) if nzo.precheck: if found and not killed: # Pre-check, proper article found, just register if sabnzbd.LOG_ALL: logging.debug('Server %s has article %s', article.fetcher, art_id) register = True elif not killed and not found: logme = T('Badly formed yEnc article in %s') % art_id logging.info(logme) if not found or killed: new_server_found = self.search_new_server(article) if new_server_found: register = False logme = None except:
def special_rss_site(url): """ Return True if url describes an RSS site with odd titles """ return cfg.rss_filenames() or match_str(url, cfg.rss_odd_titles())
class Decoder(Thread): def __init__(self, servers): Thread.__init__(self) self.queue = Queue.Queue() self.servers = servers def decode(self, article, lines): self.queue.put((article, lines)) # See if there's space left in cache, pause otherwise # But do allow some articles to enter queue, in case of full cache qsize = self.queue.qsize() if (not ArticleCache.do.reserve_space(lines) and qsize > MAX_DECODE_QUEUE) or (qsize > LIMIT_DECODE_QUEUE): sabnzbd.downloader.Downloader.do.delay() def stop(self): self.queue.put(None) def run(self): from sabnzbd.nzbqueue import NzbQueue while 1: # Sleep to allow decoder/assembler switching sleep(0.001) art_tup = self.queue.get() if not art_tup: break article, lines = art_tup nzf = article.nzf nzo = nzf.nzo art_id = article.article killed = False # Check if the space that's now free can let us continue the queue? qsize = self.queue.qsize() if (ArticleCache.do.free_reserve_space(lines) or qsize < MAX_DECODE_QUEUE) and ( qsize < LIMIT_DECODE_QUEUE ) and sabnzbd.downloader.Downloader.do.delayed: sabnzbd.downloader.Downloader.do.undelay() data = None register = True # Finish article found = False # Proper article found logme = None if lines: try: if nzo.precheck: raise BadYenc register = True logging.debug("Decoding %s", art_id) data = decode(article, lines) nzf.article_count += 1 found = True except IOError, e: logme = T('Decoding %s failed') % art_id logging.warning(logme) logging.info("Traceback: ", exc_info=True) sabnzbd.downloader.Downloader.do.pause() article.fetcher = None NzbQueue.do.reset_try_lists(nzf, nzo) register = False except MemoryError, e: logme = T('Decoder failure: Out of memory') logging.warning(logme) anfo = sabnzbd.articlecache.ArticleCache.do.cache_info() logging.info("Decoder-Queue: %d, Cache: %d, %d, %d", self.queue.qsize(), anfo.article_sum, anfo.cache_size, anfo.cache_limit) logging.info("Traceback: ", exc_info=True) sabnzbd.downloader.Downloader.do.pause() article.fetcher = None NzbQueue.do.reset_try_lists(nzf, nzo) register = False except CrcError, e: logme = T('CRC Error in %s (%s -> %s)') % ( art_id, e.needcrc, e.gotcrc) logging.info(logme) data = e.data except BadYenc: # Handles precheck and badly formed articles killed = False found = False if nzo.precheck and lines and lines[0].startswith('223 '): # STAT was used, so we only get a status code found = True else: # Examine headers (for precheck) or body (for download) # And look for DMCA clues (while skipping "X-" headers) for line in lines: lline = line.lower() if 'message-id:' in lline: found = True if not line.startswith('X-') and match_str( lline, ('dmca', 'removed', 'cancel', 'blocked')): killed = True break if killed: logme = 'Article removed from server (%s)' logging.info(logme, art_id) if nzo.precheck: if found and not killed: # Pre-check, proper article found, just register logging.debug('Server has article %s', art_id) register = True elif not killed and not found: logme = T('Badly formed yEnc article in %s') % art_id logging.info(logme) if not found or killed: new_server_found = self.__search_new_server(article) if new_server_found: register = False logme = None
def run(self): from sabnzbd.nzbqueue import NzbQueue while 1: # Sleep to allow decoder/assembler switching sleep(0.001) art_tup = self.queue.get() if not art_tup: break article, lines = art_tup nzf = article.nzf nzo = nzf.nzo art_id = article.article killed = False # Check if the space that's now free can let us continue the queue? if (ArticleCache.do.free_reserve_space(lines) or self.queue.qsize() < MAX_DECODE_QUEUE) and sabnzbd.downloader.Downloader.do.delayed: sabnzbd.downloader.Downloader.do.undelay() data = None register = True # Finish article found = False # Proper article found logme = None if lines: try: if nzo.precheck: raise BadYenc register = True logging.debug("Decoding %s", art_id) data = decode(article, lines) nzf.article_count += 1 found = True except IOError, e: logme = T('Decoding %s failed') % art_id logging.warning(logme) logging.info("Traceback: ", exc_info=True) sabnzbd.downloader.Downloader.do.pause() article.fetcher = None NzbQueue.do.reset_try_lists(nzf, nzo) register = False except CrcError, e: logme = T('CRC Error in %s (%s -> %s)') % (art_id, e.needcrc, e.gotcrc) logging.info(logme) data = e.data except BadYenc: # Handles precheck and badly formed articles killed = False found = False if nzo.precheck and lines and lines[0].startswith('223 '): # STAT was used, so we only get a status code found = True else: # Examine headers (for precheck) or body (for download) # And look for DMCA clues (while skipping "X-" headers) for line in lines: lline = line.lower() if 'message-id:' in lline: found = True if not line.startswith('X-') and match_str(lline, ('dmca', 'removed', 'cancel', 'blocked')): killed = True break if killed: logme = 'Article removed from server (%s)' logging.info(logme, art_id) if nzo.precheck: if found and not killed: # Pre-check, proper article found, just register logging.debug('Server has article %s', art_id) register = True elif not killed and not found: logme = T('Badly formed yEnc article in %s') % art_id logging.info(logme) if not found or killed: new_server_found = self.__search_new_server(article) if new_server_found: register = False logme = None
def run(self): from sabnzbd.nzbqueue import NzbQueue while 1: sleep(0.001) art_tup = self.queue.get() if not art_tup: break if self.queue.qsize( ) < MIN_DECODE_QUEUE and sabnzbd.downloader.Downloader.do.delayed: sabnzbd.downloader.Downloader.do.undelay() article, lines = art_tup nzf = article.nzf nzo = nzf.nzo art_id = article.article killed = False data = None register = True # Finish article found = False # Proper article found if lines: logme = None try: if nzo.precheck: raise BadYenc register = True logging.debug("Decoding %s", art_id) data = decode(article, lines) nzf.article_count += 1 found = True except IOError, e: logme = Ta('Decoding %s failed') % art_id logging.warning(logme) logging.info("Traceback: ", exc_info=True) sabnzbd.downloader.Downloader.do.pause() article.fetcher = None NzbQueue.do.reset_try_lists(nzf, nzo) register = False except CrcError, e: logme = Ta('CRC Error in %s (%s -> %s)') % ( art_id, e.needcrc, e.gotcrc) logging.info(logme) data = e.data if cfg.fail_on_crc(): new_server_found = self.__search_new_server(article) if new_server_found: register = False logme = None except BadYenc: # Handles precheck and badly formed articles killed = False found = False if nzo.precheck and lines and lines[0].startswith('223 '): # STAT was used, so we only get a status code found = True else: # Examine headers (for precheck) or body (for download) # And look for DMCA clues (while skipping "X-" headers) for line in lines: lline = line.lower() if 'message-id:' in lline: found = True if not line.startswith('X-') and match_str( lline, ('dmca', 'removed', 'cancel', 'blocked')): killed = True break if killed: logme = 'Article removed from server (%s)' logging.info(logme, art_id) if nzo.precheck: if found and not killed: # Pre-check, proper article found, just register logging.debug('Server has article %s', art_id) register = True elif not killed and not found: logme = Ta('Badly formed yEnc article in %s') % art_id logging.info(logme) if not found or killed: new_server_found = self.__search_new_server(article) if new_server_found: register = False logme = None
def run(self): from sabnzbd.nzbqueue import NzbQueue while 1: sleep(0.001) art_tup = self.queue.get() if not art_tup: break if self.queue.qsize() < MIN_DECODE_QUEUE and sabnzbd.downloader.Downloader.do.delayed: sabnzbd.downloader.Downloader.do.undelay() article, lines = art_tup nzf = article.nzf nzo = nzf.nzo data = None register = True # Finish article found = False # Proper article found if lines: logme = None try: if nzo.precheck: raise BadYenc register = True logging.debug("Decoding %s", article) data = decode(article, lines) nzf.article_count += 1 found = True except IOError, e: logme = Ta('Decoding %s failed') % article logging.info(logme) sabnzbd.downloader.Downloader.do.pause() article.fetcher = None NzbQueue.do.reset_try_lists(nzf, nzo) register = False except CrcError, e: logme = Ta('CRC Error in %s (%s -> %s)') % (article, e.needcrc, e.gotcrc) logging.info(logme) data = e.data if cfg.fail_on_crc(): new_server_found = self.__search_new_server(article) if new_server_found: register = False logme = None except BadYenc: # Handles precheck and badly formed articles killed = False found = False if nzo.precheck and lines and lines[0].startswith('223 '): # STAT was used, so we only get a status code found = True else: # Examine headers (for precheck) or body (for download) # And look for DMCA clues (while skipping "X-" headers) for line in lines: if not line.startswith('X-') and match_str(line.lower(), ('dmca', 'removed', 'cancel', 'blocked')): logging.info('Article removed from server (%s)', article) killed = True break if nzo.precheck: if found or not killed: # Pre-check, proper article found, just register logging.debug('Server has article %s', article) register = True elif not killed and not found: logme = Ta('Badly formed yEnc article in %s') % article logging.info(logme) if not found: new_server_found = self.__search_new_server(article) if new_server_found: register = False logme = None
def run(self): while 1: # Sleep to allow decoder/assembler switching sleep(0.0001) art_tup = self.queue.get() if not art_tup: break article, lines, raw_data = art_tup nzf = article.nzf nzo = nzf.nzo art_id = article.article killed = False # Check if the space that's now free can let us continue the queue? qsize = self.queue.qsize() if (sabnzbd.articlecache.ArticleCache.do.free_reserve_space(lines) or qsize < MAX_DECODE_QUEUE) and \ (qsize < LIMIT_DECODE_QUEUE) and sabnzbd.downloader.Downloader.do.delayed: sabnzbd.downloader.Downloader.do.undelay() data = None register = True # Finish article found = False # Proper article found logme = None if lines or raw_data: try: if nzo.precheck: raise BadYenc register = True if self.__log_decoding: logging.debug("Decoding %s", art_id) data = self.decode(article, lines, raw_data) nzf.article_count += 1 found = True except IOError: logme = T('Decoding %s failed') % art_id logging.warning(logme) logging.info("Traceback: ", exc_info=True) sabnzbd.downloader.Downloader.do.pause() sabnzbd.nzbqueue.NzbQueue.do.reset_try_lists(article) register = False except MemoryError: logme = T('Decoder failure: Out of memory') logging.warning(logme) anfo = sabnzbd.articlecache.ArticleCache.do.cache_info() logging.info("Decoder-Queue: %d, Cache: %d, %d, %d", self.queue.qsize(), anfo.article_sum, anfo.cache_size, anfo.cache_limit) logging.info("Traceback: ", exc_info=True) sabnzbd.downloader.Downloader.do.pause() sabnzbd.nzbqueue.NzbQueue.do.reset_try_lists(article) register = False except CrcError, e: logme = 'CRC Error in %s' % art_id logging.info(logme) data = e.data except (BadYenc, ValueError): # Handles precheck and badly formed articles killed = False found = False data_to_check = lines or raw_data if nzo.precheck and data_to_check and data_to_check[0].startswith('223 '): # STAT was used, so we only get a status code found = True else: # Examine headers (for precheck) or body (for download) # And look for DMCA clues (while skipping "X-" headers) for line in data_to_check: lline = line.lower() if 'message-id:' in lline: found = True if not line.startswith('X-') and match_str(lline, ('dmca', 'removed', 'cancel', 'blocked')): killed = True break if killed: logme = 'Article removed from server (%s)' logging.info(logme, art_id) if nzo.precheck: if found and not killed: # Pre-check, proper article found, just register logging.debug('Server %s has article %s', article.fetcher, art_id) register = True elif not killed and not found: logme = T('Badly formed yEnc article in %s') % art_id logging.info(logme) if not found or killed: new_server_found = self.search_new_server(article) if new_server_found: register = False logme = None except:
def run(self): while 1: # Set Article and NzbObject objects to None so references from this # thread do not keep the parent objects alive (see #1628) decoded_data = raw_data = article = nzo = None article, raw_data = self.decoder_queue.get() if not article: logging.info("Shutting down decoder %s", self.name) break nzo = article.nzf.nzo art_id = article.article # Free space in the decoder-queue sabnzbd.ArticleCache.free_reserved_space(article.bytes) # Keeping track article_success = False try: if nzo.precheck: raise BadYenc if sabnzbd.LOG_ALL: logging.debug("Decoding %s", art_id) decoded_data = decode(article, raw_data) article_success = True except MemoryError: logging.warning(T("Decoder failure: Out of memory")) logging.info("Decoder-Queue: %d", self.decoder_queue.qsize()) logging.info("Cache: %d, %d, %d", *sabnzbd.ArticleCache.cache_info()) logging.info("Traceback: ", exc_info=True) sabnzbd.Downloader.pause() # This article should be fetched again sabnzbd.NzbQueue.reset_try_lists(article) continue except CrcError as crc_error: logging.info("CRC Error in %s" % art_id) # Continue to the next one if we found new server if search_new_server(article): continue # Store data, maybe par2 can still fix it decoded_data = crc_error.data except (BadYenc, ValueError): # Handles precheck and badly formed articles if nzo.precheck and raw_data and raw_data[0].startswith( b"223 "): # STAT was used, so we only get a status code article_success = True else: # Examine headers (for precheck) or body (for download) # Look for DMCA clues (while skipping "X-" headers) # Detect potential UUencode for line in raw_data: lline = line.lower() if b"message-id:" in lline: article_success = True if not lline.startswith(b"x-") and match_str( lline, (b"dmca", b"removed", b"cancel", b"blocked")): article_success = False logging.info("Article removed from server (%s)", art_id) break if lline.find(b"\nbegin ") >= 0: logme = T( "UUencode detected, only yEnc encoding is supported [%s]" ) % nzo.final_name logging.error(logme) nzo.fail_msg = logme sabnzbd.NzbQueue.end_job(nzo) break # Pre-check, proper article found so just register if nzo.precheck and article_success and sabnzbd.LOG_ALL: logging.debug("Server %s has article %s", article.fetcher, art_id) elif not article_success: # If not pre-check, this must be a bad article if not nzo.precheck: logging.info("Badly formed yEnc article in %s", art_id, exc_info=True) # Continue to the next one if we found new server if search_new_server(article): continue except: logging.warning(T("Unknown Error while decoding %s"), art_id) logging.info("Traceback: ", exc_info=True) # Continue to the next one if we found new server if search_new_server(article): continue if decoded_data: # If the data needs to be written to disk due to full cache, this will be slow # Causing the decoder-queue to fill up and delay the downloader sabnzbd.ArticleCache.save_article(article, decoded_data) sabnzbd.NzbQueue.register_article(article, article_success)