def ProcessSingleFile(filename, path, pp=None, script=None, cat=None, catdir=None, keep=False, priority=None, nzbname=None, reuse=False, nzo_info=None, dup_check=True, url=''): """ Analyse file and create a job from it Supports NZB, NZB.GZ and GZ.NZB-in-disguise returns: -2==Error/retry, -1==Error, 0==OK, 1==OK-but-ignorecannot-delete """ from sabnzbd.nzbqueue import add_nzo if catdir is None: catdir = cat try: f = open(path, 'rb') b1 = f.read(1) b2 = f.read(1) f.close() if (b1 == '\x1f' and b2 == '\x8b'): # gzip file or gzip in disguise name = filename.replace('.nzb.gz', '.nzb') f = gzip.GzipFile(path, 'rb') else: name = filename f = open(path, 'rb') data = f.read() f.close() except: logging.warning(Ta('Cannot read %s'), path) logging.info("Traceback: ", exc_info = True) return -2 if name: name, cat = name_to_cat(name, catdir) # The name is used as the name of the folder, so sanitize it using folder specific santization name = misc.sanitize_foldername(name) try: nzo = nzbstuff.NzbObject(name, 0, pp, script, data, cat=cat, priority=priority, nzbname=nzbname, nzo_info=nzo_info, reuse=reuse, dup_check=dup_check) nzo.url = url except TypeError: # Duplicate, ignore nzo = None except: if data.find("<nzb") >= 0 and data.find("</nzb") < 0: # Looks like an incomplete file, retry return -2 else: return -1 if nzo: add_nzo(nzo) try: if not keep: os.remove(path) except: logging.error(Ta('Error removing %s'), path) logging.info("Traceback: ", exc_info = True) return 1 return 0
def _HandleLink( jobs, link, title, flag, orgcat, cat, pp, script, download, star, order, priority=NORMAL_PRIORITY, rule=0 ): """ Process one link """ if script == "": script = None if pp == "": pp = None jobs[link] = {} jobs[link]["order"] = order jobs[link]["orgcat"] = orgcat if special_rss_site(link): nzbname = None else: nzbname = sanitize_foldername(title) m = RE_NEWZBIN.search(link) if m and m.group(1).lower() == "newz" and m.group(2) and m.group(3): if download: jobs[link]["status"] = "D" jobs[link]["title"] = title logging.info("Adding %s (%s) to queue", m.group(3), title) sabnzbd.add_msgid(m.group(3), pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname) else: if star: jobs[link]["status"] = flag + "*" else: jobs[link]["status"] = flag jobs[link]["title"] = title jobs[link]["url"] = m.group(3) jobs[link]["cat"] = cat jobs[link]["pp"] = pp jobs[link]["script"] = script jobs[link]["prio"] = str(priority) else: if download: jobs[link]["status"] = "D" jobs[link]["title"] = title logging.info("Adding %s (%s) to queue", link, title) sabnzbd.add_url(link, pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname) else: if star: jobs[link]["status"] = flag + "*" else: jobs[link]["status"] = flag jobs[link]["title"] = title jobs[link]["url"] = link jobs[link]["cat"] = cat jobs[link]["pp"] = pp jobs[link]["script"] = script jobs[link]["prio"] = str(priority) jobs[link]["time"] = time.time() jobs[link]["rule"] = rule
def _HandleLink(jobs, link, title, flag, orgcat, cat, pp, script, download, star, order, priority=NORMAL_PRIORITY, rule=0): """ Process one link """ if script == '': script = None if pp == '': pp = None jobs[link] = {} jobs[link]['order'] = order jobs[link]['orgcat'] = orgcat if special_rss_site(link): nzbname = None else: nzbname = sanitize_foldername(title) m = RE_NEWZBIN.search(link) if m and m.group(1).lower() == 'newz' and m.group(2) and m.group(3): if download: jobs[link]['status'] = 'D' jobs[link]['title'] = title logging.info("Adding %s (%s) to queue", m.group(3), title) sabnzbd.add_msgid(m.group(3), pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname) else: if star: jobs[link]['status'] = flag + '*' else: jobs[link]['status'] = flag jobs[link]['title'] = title jobs[link]['url'] = m.group(3) jobs[link]['cat'] = cat jobs[link]['pp'] = pp jobs[link]['script'] = script jobs[link]['prio'] = str(priority) else: if download: jobs[link]['status'] = 'D' jobs[link]['title'] = title logging.info("Adding %s (%s) to queue", link, title) sabnzbd.add_url(link, pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname) else: if star: jobs[link]['status'] = flag + '*' else: jobs[link]['status'] = flag jobs[link]['title'] = title jobs[link]['url'] = link jobs[link]['cat'] = cat jobs[link]['pp'] = pp jobs[link]['script'] = script jobs[link]['prio'] = str(priority) jobs[link]['time'] = time.time() jobs[link]['rule'] = rule
def eval_sort(sorttype, expression, name=None, multipart=''): """ Preview a sort expression, to be used by API """ from sabnzbd.api import Ttemplate path = '' name = sanitize_foldername(name) if sorttype == 'series': name = name or ('%s S01E05 - %s [DTS]' % (Ttemplate('show-name'), Ttemplate('ep-name'))) sorter = SeriesSorter(None, name, path, 'tv') elif sorttype == 'movie': name = name or (Ttemplate('movie-sp-name') + ' (2009)') sorter = MovieSorter(None, name, path, 'tv') elif sorttype == 'date': name = name or (Ttemplate('show-name') + ' 2009-01-02') sorter = DateSorter(None, name, path, 'tv') else: return None sorter.sort_string = expression sorter.match(force=True) path = sorter.get_final_path() path = os.path.normpath(os.path.join(path, sorter.filename_set)) fname = Ttemplate('orgFilename') fpath = path if sorttype == 'movie' and '%1' in multipart: fname = fname + multipart.replace('%1', '1') fpath = fpath + multipart.replace('%1', '1') if '%fn' in path: path = path.replace('%fn', fname + '.mkv') else: if sorter.rename_or_not: path = fpath + '.mkv' else: if sabnzbd.WIN32: path += '\\' else: path += '/' return path
def run(self): logging.info('URLGrabber starting up') self.shutdown = False while not self.shutdown: # Don't pound the website! time.sleep(5.0) (url, future_nzo) = self.queue.get() if not url: # stop signal, go test self.shutdown continue if future_nzo and future_nzo.wait and future_nzo.wait > time.time(): # Requeue when too early and still active self.add(url, future_nzo) continue url = url.replace(' ', '') try: del_bookmark = not future_nzo if future_nzo: # If nzo entry deleted, give up try: deleted = future_nzo.deleted except AttributeError: deleted = True if deleted: logging.debug('Dropping URL %s, job entry missing', url) continue # Add nzbmatrix credentials if needed url, matrix_id = _matrix_url(url) # _grab_url cannot reside in a function, because the tempfile # would not survive the end of the function if del_bookmark: logging.info('Removing nzbmatrix bookmark %s', matrix_id) else: logging.info('Grabbing URL %s', url) opener = urllib.FancyURLopener({}) opener.prompt_user_passwd = None opener.addheaders = [] opener.addheader('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__) if not [True for item in _BAD_GZ_HOSTS if item in url]: opener.addheader('Accept-encoding','gzip') filename = None category = None length = 0 nzo_info = {} try: fn, header = opener.retrieve(url) except: fn = None if fn: for tup in header.items(): try: item = tup[0].lower() value = tup[1].strip() except: continue if item in ('category_id', 'x-dnzb-category'): category = value elif item in ('x-dnzb-moreinfo',): nzo_info['more_info'] = value elif item in ('x-dnzb-name',): filename = value if not filename.endswith('.nzb'): filename += '.nzb' elif item in ('content-length',): length = misc.int_conv(value) if not filename: for item in tup: if "filename=" in item: filename = item[item.index("filename=") + 9:].strip(';').strip('"') if matrix_id: fn, msg, retry, wait = _analyse_matrix(fn, matrix_id) if not fn: if retry: logging.info(msg) logging.debug('Retry nzbmatrix item %s after waiting %s sec', matrix_id, wait) self.add(url, future_nzo, wait) else: logging.error(msg) misc.bad_fetch(future_nzo, clean_matrix_url(url), msg, retry=True) continue category = _MATRIX_MAP.get(category, category) if del_bookmark: # No retries of nzbmatrix bookmark removals continue else: fn, msg, retry, wait = _analyse_others(fn, url) if not fn: if retry: logging.info('Retry URL %s', url) self.add(url, future_nzo, wait) else: misc.bad_fetch(future_nzo, url, msg, retry=True) continue if not filename: filename = os.path.basename(url) + '.nzb' # Sanitize and trim name, preserving the extension filename, ext = os.path.splitext(filename) filename = misc.sanitize_foldername(filename) filename += '.' + misc.sanitize_foldername(ext) pp = future_nzo.pp script = future_nzo.script cat = future_nzo.cat if (cat is None or cat == '*') and category: cat = misc.cat_convert(category) priority = future_nzo.priority nzbname = future_nzo.custom_name # Check if nzb file if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz'): res, nzo_ids = dirscanner.ProcessSingleFile(filename, fn, pp=pp, script=script, cat=cat, priority=priority, \ nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url) if res == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) else: if res == -2: logging.info('Incomplete NZB, retry after 5 min %s', url) when = 300 else: logging.info('Unknown error fetching NZB, retry after 2 min %s', url) when = 120 self.add(url, future_nzo, when) # Check if a supported archive else: if dirscanner.ProcessArchiveFile(filename, fn, pp, script, cat, priority=priority, url=future_nzo.url)[0] == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) else: # Not a supported filetype, not an nzb (text/html ect) try: os.remove(fn) except: pass logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url) self.add(url, future_nzo, 30) except: logging.error('URLGRABBER CRASHED', exc_info=True) logging.debug("URLGRABBER Traceback: ", exc_info=True)
def ProcessArchiveFile(filename, path, pp=None, script=None, cat=None, catdir=None, keep=False, priority=None, url='', nzbname=None): """ Analyse ZIP file and create job(s). Accepts ZIP files with ONLY nzb/nfo/folder files in it. returns (status, nzo_ids) status: -1==Error/Retry, 0==OK, 1==Ignore """ from sabnzbd.nzbqueue import add_nzo nzo_ids = [] if catdir is None: catdir = cat filename, cat = name_to_cat(filename, catdir) if zipfile.is_zipfile(path): try: zf = zipfile.ZipFile(path) except: return -1, [] elif is_rarfile(path): try: zf = RarFile(path) except: return -1, [] else: return 1, [] status = 1 names = zf.namelist() names.sort() nzbcount = 0 for name in names: name = name.lower() if not (name.endswith('.nzb') or name.endswith('.nfo') or name.endswith('/')): status = 1 break elif name.endswith('.nzb'): status = 0 nzbcount += 1 if status == 0: if nzbcount != 1: nzbname = None for name in names: if name.lower().endswith('.nzb'): try: data = zf.read(name) except: zf.close() return -1, [] name = re.sub(r'\[.*nzbmatrix.com\]', '', name) name = os.path.basename(name) name = misc.sanitize_foldername(name) if data: try: nzo = nzbstuff.NzbObject(name, 0, pp, script, data, cat=cat, url=url, priority=priority, nzbname=nzbname) except: nzo = None if nzo: nzo_ids.append(add_nzo(nzo)) nzo.update_rating() zf.close() try: if not keep: os.remove(path) except: logging.error(Ta('Error removing %s'), path) logging.info("Traceback: ", exc_info = True) status = 1 else: zf.close() status = 1 return status, nzo_ids
def ProcessSingleFile(filename, path, pp=None, script=None, cat=None, catdir=None, keep=False, priority=None, nzbname=None, reuse=False, nzo_info=None, dup_check=True, url=''): """ Analyse file and create a job from it Supports NZB, NZB.GZ and GZ.NZB-in-disguise returns (status, nzo_ids) status: -2==Error/retry, -1==Error, 0==OK, 1==OK-but-ignorecannot-delete """ from sabnzbd.nzbqueue import add_nzo nzo_ids = [] if catdir is None: catdir = cat try: f = open(path, 'rb') b1 = f.read(1) b2 = f.read(1) f.close() if (b1 == '\x1f' and b2 == '\x8b'): # gzip file or gzip in disguise name = filename.replace('.nzb.gz', '.nzb') f = gzip.GzipFile(path, 'rb') else: name = filename f = open(path, 'rb') data = f.read() f.close() except: logging.warning(Ta('Cannot read %s'), path) logging.info("Traceback: ", exc_info = True) return -2, nzo_ids if name: name, cat = name_to_cat(name, catdir) # The name is used as the name of the folder, so sanitize it using folder specific santization if not nzbname: # Prevent embedded password from being damaged by sanitize and trimming nzbname = os.path.split(name)[1] name = misc.sanitize_foldername(name) try: nzo = nzbstuff.NzbObject(name, 0, pp, script, data, cat=cat, priority=priority, nzbname=nzbname, nzo_info=nzo_info, url=url, reuse=reuse, dup_check=dup_check) except TypeError: # Duplicate, ignore nzo = None except ValueError: # Empty, but correct file return -1, nzo_ids except: if data.find("<nzb") >= 0 and data.find("</nzb") < 0: # Looks like an incomplete file, retry return -2, nzo_ids else: return -1, nzo_ids if nzo: nzo_ids.append(add_nzo(nzo)) nzo.update_rating() try: if not keep: os.remove(path) except: logging.error(Ta('Error removing %s'), path) logging.info("Traceback: ", exc_info = True) return 1, nzo_ids return 0, nzo_ids
def _grabnzb(msgid): """ Grab one msgid from newzbin """ msg = '' retry = (60, None, None, None) nzo_info = {'msgid': msgid} logging.info('Fetching NZB for Newzbin report #%s', msgid) headers = {'User-agent' : 'SABnzbd+/%s' % sabnzbd.version.__version__} # Connect to Newzbin try: if _HAVE_SSL: conn = httplib.HTTPSConnection(cfg.newzbin_url()) else: conn = httplib.HTTPConnection(cfg.newzbin_url()) postdata = { 'username': cfg.newzbin_username(), 'password': cfg.newzbin_password(), 'reportid': msgid } postdata = urllib.urlencode(postdata) headers['Content-type'] = 'application/x-www-form-urlencoded' fetchurl = '/api/dnzb/' conn.request('POST', fetchurl, postdata, headers) response = conn.getresponse() # Save debug info if we have to data = response.read() except: _warn_user('Problem accessing Newzbin server, wait 1 min.') logging.info("Traceback: ", exc_info = True) return retry # Get the filename rcode = response.getheader('X-DNZB-RCode') rtext = response.getheader('X-DNZB-RText') try: nzo_info['more_info'] = response.getheader('X-DNZB-MoreInfo') except: # Only some reports will generate a moreinfo header pass if not (rcode or rtext): logging.error(T('Newzbin server changed its protocol')) return retry # Official return codes: # 200 = OK, NZB content follows # 400 = Bad Request, please supply all parameters # (this generally means reportid or fileid is missing; missing user/pass gets you a 401) # 401 = Unauthorised, check username/password? # 402 = Payment Required, not Premium # 404 = Not Found, data doesn't exist? # (only working for reportids, see Technical Limitations) # 450 = Try Later, wait <x> seconds for counter to reset # (for an explanation of this, see DNZB Rate Limiting) # 500 = Internal Server Error, please report to Administrator # 503 = Service Unavailable, site is currently down if rcode in ('500', '503'): _warn_user('Newzbin has a server problem (%s, %s), wait 5 min.' % (rcode, rtext)) return retry _access_ok() if rcode == '450': wait_re = re.compile('wait (\d+) seconds') try: wait = int(wait_re.findall(rtext)[0]) except: wait = 60 if wait > 60: wait = 60 logging.debug("Newzbin says we should wait for %s sec", wait) return int(wait+1), None, None, None if rcode in ('402'): msg = Ta('You have no credit on your Newzbin account') return None, None, None, msg if rcode in ('401'): msg = Ta('Unauthorised, check your newzbin username/password') return None, None, None, msg if rcode in ('400', '404'): msg = Ta('Newzbin report %s not found') % msgid return None, None, None, msg if rcode != '200': msg = Ta('Newzbin gives undocumented error code (%s, %s)') % (rcode, rtext) return 60, None, None, msg # Process data report_name = response.getheader('X-DNZB-Name') report_cat = response.getheader('X-DNZB-Category') if not (report_name and report_cat): msg = Ta('Newzbin server fails to give info for %s') % msgid return 60, None, None, msg # sanitize report_name newname = sanitize_foldername(report_name) if len(newname) > 80: newname = newname[0:79].strip('. ') newname += ".nzb" logging.info('Successfully fetched report %s - %s (cat=%s) (%s)', msgid, report_name, report_cat, newname) return (newname, data, report_cat, nzo_info)
def run(self): logging.info('URLGrabber starting up') self.shutdown = False while not self.shutdown: # Don't pound the website! time.sleep(5.0) (url, future_nzo, retry_count) = self.queue.get() if not url: continue url = url.replace(' ', '') try: del_bookmark = not future_nzo if future_nzo: # If nzo entry deleted, give up try: deleted = future_nzo.deleted except: deleted = True if deleted: logging.debug('Dropping URL %s, job entry missing', url) continue # Add nzbmatrix credentials if needed url, matrix_id = _matrix_url(url) # When still waiting for nzbmatrix wait period, requeue if matrix_id and self.matrix_wait > time.time(): self.queue.put((url, future_nzo, retry_count)) continue # _grab_url cannot reside in a function, because the tempfile # would not survive the end of the function if del_bookmark: logging.info('Removing nzbmatrix bookmark %s', matrix_id) else: logging.info('Grabbing URL %s', url) opener = urllib.FancyURLopener({}) opener.prompt_user_passwd = None opener.addheaders = [] opener.addheader('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__) opener.addheader('Accept-encoding', 'gzip') filename = None category = None length = 0 nzo_info = {} try: fn, header = opener.retrieve(url) except: fn = None if fn: for tup in header.items(): try: item = tup[0].lower() value = tup[1].strip() except: continue if item in ('category_id', 'x-dnzb-category'): category = value elif item in ('x-dnzb-moreinfo', ): nzo_info['more_info'] = value elif item in ('x-dnzb-name', ): filename = value if not filename.endswith('.nzb'): filename += '.nzb' elif item in ('content-length', ): length = misc.int_conv(value) if not filename: for item in tup: if "filename=" in item: filename = item[item.index("filename=") + 9:].strip(';').strip('"') if matrix_id: fn, msg, retry, wait = _analyse_matrix(fn, matrix_id) if retry and wait > 0: self.matrix_wait = time.time() + wait logging.debug('Retry URL %s after waiting', url) self.queue.put((url, future_nzo, retry_count)) continue category = _MATRIX_MAP.get(category, category) else: msg = '' retry = True # Check if the filepath is specified, if not, check if a retry is allowed. if not fn: retry_count -= 1 if retry_count > 0 and retry: logging.info('Retry URL %s', url) self.queue.put((url, future_nzo, retry_count)) elif not del_bookmark: misc.bad_fetch(future_nzo, url, msg, retry=True) continue if del_bookmark: continue if not filename: filename = os.path.basename(url) + '.nzb' # Sanitize and trim name, preserving the extension filename, ext = os.path.splitext(filename) filename = misc.sanitize_foldername(filename) filename += '.' + misc.sanitize_foldername(ext) pp = future_nzo.pp script = future_nzo.script cat = future_nzo.cat if (cat is None or cat == '*') and category: cat = misc.cat_convert(category) priority = future_nzo.priority nzbname = future_nzo.custom_name # Check if nzb file if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz'): res = dirscanner.ProcessSingleFile(filename, fn, pp=pp, script=script, cat=cat, priority=priority, \ nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url) if res == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) elif res == -2: retry_count -= 1 if retry_count > 0: logging.info('Incomplete NZB, retry %s', url) self.queue.put((url, future_nzo, retry_count)) else: misc.bad_fetch(future_nzo, url, retry=True, content=True) else: misc.bad_fetch(future_nzo, url, retry=True, content=True) # Check if a supported archive else: if dirscanner.ProcessArchiveFile(filename, fn, pp, script, cat, priority=priority, url=future_nzo.url) == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) else: # Not a supported filetype, not an nzb (text/html ect) try: os.remove(fn) except: pass misc.bad_fetch(future_nzo, url, retry=True, content=True) except: logging.error('URLGRABBER CRASHED', exc_info=True) logging.debug("URLGRABBER Traceback: ", exc_info=True)
def _grabnzb(msgid): """ Grab one msgid from newzbin """ msg = '' retry = (60, None, None, None) nzo_info = {'msgid': msgid} logging.info('Fetching NZB for Newzbin report #%s', msgid) headers = {'User-agent' : 'SABnzbd+/%s' % sabnzbd.version.__version__} # Connect to Newzbin try: if _HAVE_SSL: conn = httplib.HTTPSConnection('www.newzbin.com') else: conn = httplib.HTTPConnection('www.newzbin.com') postdata = { 'username': cfg.newzbin_username(), 'password': cfg.newzbin_password(), 'reportid': msgid } postdata = urllib.urlencode(postdata) headers['Content-type'] = 'application/x-www-form-urlencoded' fetchurl = '/api/dnzb/' conn.request('POST', fetchurl, postdata, headers) response = conn.getresponse() # Save debug info if we have to data = response.read() except: _warn_user('Problem accessing Newzbin server, wait 1 min.') logging.info("Traceback: ", exc_info = True) return retry # Get the filename rcode = response.getheader('X-DNZB-RCode') rtext = response.getheader('X-DNZB-RText') try: nzo_info['more_info'] = response.getheader('X-DNZB-MoreInfo') except: # Only some reports will generate a moreinfo header pass if not (rcode or rtext): logging.error(T('Newzbin server changed its protocol')) return None, None, None, None # Official return codes: # 200 = OK, NZB content follows # 400 = Bad Request, please supply all parameters # (this generally means reportid or fileid is missing; missing user/pass gets you a 401) # 401 = Unauthorised, check username/password? # 402 = Payment Required, not Premium # 404 = Not Found, data doesn't exist? # (only working for reportids, see Technical Limitations) # 450 = Try Later, wait <x> seconds for counter to reset # (for an explanation of this, see DNZB Rate Limiting) # 500 = Internal Server Error, please report to Administrator # 503 = Service Unavailable, site is currently down if rcode in ('500', '503'): _warn_user('Newzbin has a server problem (%s, %s), wait 5 min.' % (rcode, rtext)) return retry _access_ok() if rcode == '450': wait_re = re.compile('wait (\d+) seconds') try: wait = int(wait_re.findall(rtext)[0]) except: wait = 60 if wait > 60: wait = 60 logging.debug("Newzbin says we should wait for %s sec", wait) return int(wait+1), None, None, None if rcode in ('402'): msg = Ta('You have no credit on your Newzbin account') return None, None, None, msg if rcode in ('401'): msg = Ta('Unauthorised, check your newzbin username/password') return None, None, None, msg if rcode in ('400', '404'): msg = Ta('Newzbin report %s not found') % msgid return None, None, None, msg if rcode != '200': msg = Ta('Newzbin gives undocumented error code (%s, %s)') % (rcode, rtext) return None, None, None, msg # Process data report_name = response.getheader('X-DNZB-Name') report_cat = response.getheader('X-DNZB-Category') if not (report_name and report_cat): msg = Ta('Newzbin server fails to give info for %s') % msgid return None, None, None, msg # sanitize report_name newname = sanitize_foldername(report_name) if len(newname) > 80: newname = newname[0:79].strip('. ') newname += ".nzb" logging.info('Successfully fetched report %s - %s (cat=%s) (%s)', msgid, report_name, report_cat, newname) return (newname, data, report_cat, nzo_info)
def run(self): logging.info('URLGrabber starting up') self.shutdown = False while not self.shutdown: # Don't pound the website! time.sleep(5.0) (url, future_nzo, retry_count) = self.queue.get() if not url: continue url = url.replace(' ', '') try: del_bookmark = not future_nzo if future_nzo: # If nzo entry deleted, give up try: deleted = future_nzo.deleted except: deleted = True if deleted: logging.debug('Dropping URL %s, job entry missing', url) continue # Add nzbmatrix credentials if needed url, matrix_id = _matrix_url(url) # When still waiting for nzbmatrix wait period, requeue if matrix_id and self.matrix_wait > time.time(): self.queue.put((url, future_nzo, retry_count)) continue # _grab_url cannot reside in a function, because the tempfile # would not survive the end of the function if del_bookmark: logging.info('Removing nzbmatrix bookmark %s', matrix_id) else: logging.info('Grabbing URL %s', url) opener = urllib.FancyURLopener({}) opener.prompt_user_passwd = None opener.addheaders = [] opener.addheader('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__) opener.addheader('Accept-encoding','gzip') filename = None category = None length = 0 nzo_info = {} try: fn, header = opener.retrieve(url) except: fn = None if fn: for tup in header.items(): try: item = tup[0].lower() value = tup[1].strip() except: continue if item in ('category_id', 'x-dnzb-category'): category = value elif item in ('x-dnzb-moreinfo',): nzo_info['more_info'] = value elif item in ('x-dnzb-name',): filename = value if not filename.endswith('.nzb'): filename += '.nzb' elif item in ('content-length',): length = misc.int_conv(value) if not filename: for item in tup: if "filename=" in item: filename = item[item.index("filename=") + 9:].strip(';').strip('"') if matrix_id: fn, msg, retry, wait = _analyse_matrix(fn, matrix_id) if retry and wait > 0: self.matrix_wait = time.time() + wait logging.debug('Retry URL %s after waiting', url) self.queue.put((url, future_nzo, retry_count)) continue category = _MATRIX_MAP.get(category, category) else: msg = '' retry = True # Check if the filepath is specified, if not, check if a retry is allowed. if not fn: retry_count -= 1 if retry_count > 0 and retry: logging.info('Retry URL %s', url) self.queue.put((url, future_nzo, retry_count)) elif not del_bookmark: misc.bad_fetch(future_nzo, url, msg, retry=True) continue if del_bookmark: continue if not filename: filename = os.path.basename(url) + '.nzb' # Sanitize and trim name, preserving the extension filename, ext = os.path.splitext(filename) filename = misc.sanitize_foldername(filename) filename += '.' + misc.sanitize_foldername(ext) pp = future_nzo.pp script = future_nzo.script cat = future_nzo.cat if (cat is None or cat == '*') and category: cat = misc.cat_convert(category) priority = future_nzo.priority nzbname = future_nzo.custom_name # Check if nzb file if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz'): res = dirscanner.ProcessSingleFile(filename, fn, pp=pp, script=script, cat=cat, priority=priority, \ nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url) if res == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) elif res == -2: retry_count -= 1 if retry_count > 0: logging.info('Incomplete NZB, retry %s', url) self.queue.put((url, future_nzo, retry_count)) else: misc.bad_fetch(future_nzo, url, retry=True, content=True) else: misc.bad_fetch(future_nzo, url, retry=True, content=True) # Check if a supported archive else: if dirscanner.ProcessArchiveFile(filename, fn, pp, script, cat, priority=priority, url=future_nzo.url) == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) else: # Not a supported filetype, not an nzb (text/html ect) try: os.remove(fn) except: pass misc.bad_fetch(future_nzo, url, retry=True, content=True) except: logging.error('URLGRABBER CRASHED', exc_info=True) logging.debug("URLGRABBER Traceback: ", exc_info=True)
def run(self): logging.info('URLGrabber starting up') self.shutdown = False while not self.shutdown: # Don't pound the website! time.sleep(5.0) (url, future_nzo) = self.queue.get() if not url: # stop signal, go test self.shutdown continue if future_nzo and future_nzo.wait and future_nzo.wait > time.time( ): # Requeue when too early and still active self.add(url, future_nzo) continue url = url.replace(' ', '') try: del_bookmark = not future_nzo if future_nzo: # If nzo entry deleted, give up try: deleted = future_nzo.deleted except AttributeError: deleted = True if deleted: logging.debug('Dropping URL %s, job entry missing', url) continue # Add nzbmatrix credentials if needed url, matrix_id = _matrix_url(url) # _grab_url cannot reside in a function, because the tempfile # would not survive the end of the function if del_bookmark: logging.info('Removing nzbmatrix bookmark %s', matrix_id) else: logging.info('Grabbing URL %s', url) if '.nzbsrus.' in url: opener = urllib.URLopener({}) else: opener = urllib.FancyURLopener({}) opener.prompt_user_passwd = None opener.addheaders = [] opener.addheader('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__) if not [True for item in _BAD_GZ_HOSTS if item in url]: opener.addheader('Accept-encoding', 'gzip') filename = None category = None length = 0 nzo_info = {} try: fn, header = opener.retrieve(url) except: fn = None if fn: for tup in header.items(): try: item = tup[0].lower() value = tup[1].strip() except: continue if item in ('category_id', 'x-dnzb-category'): category = value elif item in ('x-dnzb-moreinfo', ): nzo_info['more_info'] = value elif item in ('x-dnzb-name', ): filename = value if not filename.endswith('.nzb'): filename += '.nzb' elif item in ('content-length', ): length = misc.int_conv(value) if not filename: for item in tup: if "filename=" in item: filename = item[item.index("filename=") + 9:].strip(';').strip('"') if matrix_id: fn, msg, retry, wait = _analyse_matrix(fn, matrix_id) if not fn: if retry: logging.info(msg) logging.debug( 'Retry nzbmatrix item %s after waiting %s sec', matrix_id, wait) self.add(url, future_nzo, wait) else: logging.error(msg) misc.bad_fetch(future_nzo, clean_matrix_url(url), msg, retry=True) continue category = get_matrix_category(url, category) if del_bookmark: # No retries of nzbmatrix bookmark removals continue else: fn, msg, retry, wait = _analyse_others(fn, url) if not fn: if retry: logging.info('Retry URL %s', url) self.add(url, future_nzo, wait) else: misc.bad_fetch(future_nzo, url, msg, retry=True) continue if not filename: filename = os.path.basename(url) + '.nzb' # Sanitize and trim name, preserving the extension filename, ext = os.path.splitext(filename) filename = misc.sanitize_foldername(filename) filename += '.' + misc.sanitize_foldername(ext) pp = future_nzo.pp script = future_nzo.script cat = future_nzo.cat if (cat is None or cat == '*') and category: cat = misc.cat_convert(category) priority = future_nzo.priority nzbname = future_nzo.custom_name # Check if nzb file if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz'): res, nzo_ids = dirscanner.ProcessSingleFile(filename, fn, pp=pp, script=script, cat=cat, priority=priority, \ nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url) if res == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) else: if res == -2: logging.info( 'Incomplete NZB, retry after 5 min %s', url) when = 300 elif res == -1: # Error, but no reason to retry. Warning is already given NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) continue else: logging.info( 'Unknown error fetching NZB, retry after 2 min %s', url) when = 120 self.add(url, future_nzo, when) # Check if a supported archive else: if dirscanner.ProcessArchiveFile( filename, fn, pp, script, cat, priority=priority, url=future_nzo.url)[0] == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) else: # Not a supported filetype, not an nzb (text/html ect) try: os.remove(fn) except: pass logging.info( 'Unknown filetype when fetching NZB, retry after 30s %s', url) self.add(url, future_nzo, 30) except: logging.error('URLGRABBER CRASHED', exc_info=True) logging.debug("URLGRABBER Traceback: ", exc_info=True)