def application_openFiles_(self, nsapp, filenames): # logging.info('[osx] file open') # logging.info('[osx] file : %s' % (filenames)) for name in filenames: logging.info('[osx] receiving from OSX : %s', name) if os.path.exists(name): fn = get_filename(name) # logging.info('[osx] filename : %s' % (fn)) if fn: if get_ext(name) in VALID_ARCHIVES: # logging.info('[osx] archive') dirscanner.ProcessArchiveFile(fn, name, keep=True) elif get_ext(name) in ('.nzb', '.gz', '.bz2'): # logging.info('[osx] nzb') dirscanner.ProcessSingleFile(fn, name, keep=True)
def run(self): logging.info('URLGrabber starting up') self.shutdown = False while not self.shutdown: (url, future_nzo) = self.queue.get() if not url: # stop signal, go test self.shutdown continue if future_nzo: # Re-queue when too early and still active if future_nzo.wait and future_nzo.wait > time.time(): self.add(url, future_nzo) time.sleep(1.0) continue # Paused if future_nzo.status == Status.PAUSED: self.add(url, future_nzo) time.sleep(1.0) continue url = url.replace(' ', '') try: if future_nzo: # If nzo entry deleted, give up try: deleted = future_nzo.deleted except AttributeError: deleted = True if deleted: logging.debug('Dropping URL %s, job entry missing', url) continue filename = None category = None gzipped = False nzo_info = {} wait = 0 retry = True fn = None logging.info('Grabbing URL %s', url) try: fn = _build_request(url) except Exception, e: # Cannot list exceptions here, because of unpredictability over platforms error0 = str(sys.exc_info()[0]).lower() error1 = str(sys.exc_info()[1]).lower() logging.debug('Error "%s" trying to get the url %s', error1, url) if 'certificate_verify_failed' in error1 or 'certificateerror' in error0: msg = T('Server %s uses an untrusted HTTPS certificate') % '' msg += ' - https://sabnzbd.org/certificate-errors' retry = False elif 'nodename nor servname provided' in error1: msg = T('Server name does not resolve') retry = False elif '401' in error1 or 'unauthorized' in error1: msg = T('Unauthorized access') retry = False elif '404' in error1: msg = T('File not on server') retry = False elif hasattr(e, 'headers') and 'retry-after' in e.headers: # Catch if the server send retry (e.headers is case-INsensitive) wait = misc.int_conv(e.headers['retry-after']) new_url = dereferring(url, fn) if new_url: self.add(new_url, future_nzo) continue if fn: for hdr in fn.headers: try: item = hdr.lower() value = fn.headers[hdr] except: continue if item in ('content-encoding',) and value == 'gzip': gzipped = True if item in ('category_id', 'x-dnzb-category'): category = value elif item in ('x-dnzb-moreinfo',): nzo_info['more_info'] = value elif item in ('x-dnzb-name',): filename = value if not filename.endswith('.nzb'): filename += '.nzb' elif item == 'x-dnzb-propername': nzo_info['propername'] = value elif item == 'x-dnzb-episodename': nzo_info['episodename'] = value elif item == 'x-dnzb-year': nzo_info['year'] = value elif item == 'x-dnzb-failure': nzo_info['failure'] = value elif item == 'x-dnzb-details': nzo_info['details'] = value elif item == 'x-dnzb-password': nzo_info['password'] = value elif item == 'retry-after': wait = misc.int_conv(value) # Rating fields if item in _RARTING_FIELDS: nzo_info[item] = value if not filename and "filename=" in value: filename = value[value.index("filename=") + 9:].strip(';').strip('"') if wait: # For sites that have a rate-limiting attribute msg = '' retry = True fn = None elif retry: fn, msg, retry, wait, data = _analyse(fn, url) if not fn: if retry: logging.info('Retry URL %s', url) self.add(url, future_nzo, wait) else: bad_fetch(future_nzo, url, msg) continue if not filename: filename = os.path.basename(url) elif '&nzbname=' in filename: # Sometimes the filename contains the full URL, duh! filename = filename[filename.find('&nzbname=') + 9:] pp = future_nzo.pp script = future_nzo.script cat = future_nzo.cat if (cat is None or cat == '*') and category: cat = misc.cat_convert(category) priority = future_nzo.priority nzbname = future_nzo.custom_name # process data if gzipped: filename += '.gz' if not data: try: data = fn.read() except (IncompleteRead, IOError): bad_fetch(future_nzo, url, T('Server could not complete request')) fn.close() continue fn.close() if '<nzb' in data and misc.get_ext(filename) != '.nzb': filename += '.nzb' # Sanitize filename first (also removing forbidden Windows-names) filename = misc.sanitize_filename(filename) # Write data to temp file path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER) path = os.path.join(path, filename) f = open(path, 'wb') f.write(data) f.close() del data # Check if nzb file if misc.get_ext(filename) in ('.nzb', '.gz', 'bz2'): res = dirscanner.ProcessSingleFile(filename, path, pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False, nzo_id=future_nzo.nzo_id)[0] if res: if res == -2: logging.info('Incomplete NZB, retry after 5 min %s', url) when = 300 elif res == -1: # Error, but no reason to retry. Warning is already given NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) continue else: logging.info('Unknown error fetching NZB, retry after 2 min %s', url) when = 120 self.add(url, future_nzo, when) # Check if a supported archive else: status, zf, exp_ext = dirscanner.is_archive(path) if status == 0: if misc.get_ext(filename) not in ('.rar', '.zip', '.7z'): filename = filename + exp_ext os.rename(path, path + exp_ext) path = path + exp_ext dirscanner.ProcessArchiveFile(filename, path, pp, script, cat, priority=priority, nzbname=nzbname, url=future_nzo.url, keep=False, nzo_id=future_nzo.nzo_id) # Not a supported filetype, not an nzb (text/html ect) try: os.remove(fn) except: pass logging.info('Unknown filetype when fetching NZB, retry after 30s %s', url) self.add(url, future_nzo, 30) except:
def run(self): logging.info('URLGrabber starting up') self.shutdown = False while not self.shutdown: # Don't pound the website! time.sleep(5.0) (url, future_nzo, retry_count) = self.queue.get() if not url: continue url = url.replace(' ', '') try: del_bookmark = not future_nzo if future_nzo: # If nzo entry deleted, give up try: deleted = future_nzo.deleted except: deleted = True if deleted: logging.debug('Dropping URL %s, job entry missing', url) continue # Add nzbmatrix credentials if needed url, matrix_id = _matrix_url(url) # When still waiting for nzbmatrix wait period, requeue if matrix_id and self.matrix_wait > time.time(): self.queue.put((url, future_nzo, retry_count)) continue # _grab_url cannot reside in a function, because the tempfile # would not survive the end of the function if del_bookmark: logging.info('Removing nzbmatrix bookmark %s', matrix_id) else: logging.info('Grabbing URL %s', url) opener = urllib.FancyURLopener({}) opener.prompt_user_passwd = None opener.addheaders = [] opener.addheader('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__) opener.addheader('Accept-encoding', 'gzip') filename = None category = None length = 0 nzo_info = {} try: fn, header = opener.retrieve(url) except: fn = None if fn: for tup in header.items(): try: item = tup[0].lower() value = tup[1].strip() except: continue if item in ('category_id', 'x-dnzb-category'): category = value elif item in ('x-dnzb-moreinfo', ): nzo_info['more_info'] = value elif item in ('x-dnzb-name', ): filename = value if not filename.endswith('.nzb'): filename += '.nzb' elif item in ('content-length', ): length = misc.int_conv(value) if not filename: for item in tup: if "filename=" in item: filename = item[item.index("filename=") + 9:].strip(';').strip('"') if matrix_id: fn, msg, retry, wait = _analyse_matrix(fn, matrix_id) if retry and wait > 0: self.matrix_wait = time.time() + wait logging.debug('Retry URL %s after waiting', url) self.queue.put((url, future_nzo, retry_count)) continue category = _MATRIX_MAP.get(category, category) else: msg = '' retry = True # Check if the filepath is specified, if not, check if a retry is allowed. if not fn: retry_count -= 1 if retry_count > 0 and retry: logging.info('Retry URL %s', url) self.queue.put((url, future_nzo, retry_count)) elif not del_bookmark: misc.bad_fetch(future_nzo, url, msg, retry=True) continue if del_bookmark: continue if not filename: filename = os.path.basename(url) + '.nzb' # Sanitize and trim name, preserving the extension filename, ext = os.path.splitext(filename) filename = misc.sanitize_foldername(filename) filename += '.' + misc.sanitize_foldername(ext) pp = future_nzo.pp script = future_nzo.script cat = future_nzo.cat if (cat is None or cat == '*') and category: cat = misc.cat_convert(category) priority = future_nzo.priority nzbname = future_nzo.custom_name # Check if nzb file if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz'): res = dirscanner.ProcessSingleFile(filename, fn, pp=pp, script=script, cat=cat, priority=priority, \ nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url) if res == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) elif res == -2: retry_count -= 1 if retry_count > 0: logging.info('Incomplete NZB, retry %s', url) self.queue.put((url, future_nzo, retry_count)) else: misc.bad_fetch(future_nzo, url, retry=True, content=True) else: misc.bad_fetch(future_nzo, url, retry=True, content=True) # Check if a supported archive else: if dirscanner.ProcessArchiveFile(filename, fn, pp, script, cat, priority=priority, url=future_nzo.url) == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) else: # Not a supported filetype, not an nzb (text/html ect) try: os.remove(fn) except: pass misc.bad_fetch(future_nzo, url, retry=True, content=True) except: logging.error('URLGRABBER CRASHED', exc_info=True) logging.debug("URLGRABBER Traceback: ", exc_info=True)
def run(self): logging.info('URLGrabber starting up') self.shutdown = False while not self.shutdown: # Don't pound the website! time.sleep(5.0) (url, future_nzo) = self.queue.get() if not url: # stop signal, go test self.shutdown continue if future_nzo and future_nzo.wait and future_nzo.wait > time.time( ): # Requeue when too early and still active self.add(url, future_nzo) continue url = url.replace(' ', '') try: del_bookmark = not future_nzo if future_nzo: # If nzo entry deleted, give up try: deleted = future_nzo.deleted except AttributeError: deleted = True if deleted: logging.debug('Dropping URL %s, job entry missing', url) continue # Add nzbmatrix credentials if needed url, matrix_id = _matrix_url(url) # _grab_url cannot reside in a function, because the tempfile # would not survive the end of the function if del_bookmark: logging.info('Removing nzbmatrix bookmark %s', matrix_id) else: logging.info('Grabbing URL %s', url) if '.nzbsrus.' in url: opener = urllib.URLopener({}) else: opener = urllib.FancyURLopener({}) opener.prompt_user_passwd = None opener.addheaders = [] opener.addheader('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__) if not [True for item in _BAD_GZ_HOSTS if item in url]: opener.addheader('Accept-encoding', 'gzip') filename = None category = None length = 0 nzo_info = {} wait = 0 try: fn, header = opener.retrieve(url) except: fn = None if fn: for tup in header.items(): try: item = tup[0].lower() value = tup[1].strip() except: continue if item in ('category_id', 'x-dnzb-category'): category = value elif item in ('x-dnzb-moreinfo', ): nzo_info['more_info'] = value elif item in ('x-dnzb-name', ): filename = value if not filename.endswith('.nzb'): filename += '.nzb' elif item == 'x-dnzb-propername': nzo_info['propername'] = value elif item == 'x-dnzb-episodename': nzo_info['episodename'] = value elif item == 'x-dnzb-year': nzo_info['year'] = value elif item == 'x-dnzb-failure': nzo_info['failure'] = value elif item == 'x-dnzb-details': nzo_info['details'] = value elif item in ('content-length', ): length = misc.int_conv(value) elif item == 'retry-after': # For NZBFinder wait = misc.int_conv(value) if not filename: for item in tup: if "filename=" in item: filename = item[item.index("filename=") + 9:].strip(';').strip('"') if matrix_id: fn, msg, retry, wait = _analyse_matrix(fn, matrix_id) if not fn: if retry: logging.info(msg) logging.debug( 'Retry nzbmatrix item %s after waiting %s sec', matrix_id, wait) self.add(url, future_nzo, wait) else: logging.error(msg) misc.bad_fetch(future_nzo, clean_matrix_url(url), msg, retry=True) continue category = get_matrix_category(url, category) if del_bookmark: # No retries of nzbmatrix bookmark removals continue else: if wait: # For sites that have a rate-limiting attribute msg = '' retry = True fn = None else: fn, msg, retry, wait = _analyse_others(fn, url) if not fn: if retry: logging.info('Retry URL %s', url) self.add(url, future_nzo, wait) else: misc.bad_fetch(future_nzo, url, msg, retry=True) continue if not filename: filename = os.path.basename(url) + '.nzb' pp = future_nzo.pp script = future_nzo.script cat = future_nzo.cat if (cat is None or cat == '*') and category: cat = misc.cat_convert(category) priority = future_nzo.priority nzbname = future_nzo.custom_name # Check if nzb file if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz'): res, nzo_ids = dirscanner.ProcessSingleFile(filename, fn, pp=pp, script=script, cat=cat, priority=priority, \ nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url) if res == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) else: if res == -2: logging.info( 'Incomplete NZB, retry after 5 min %s', url) when = 300 elif res == -1: # Error, but no reason to retry. Warning is already given NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) continue else: logging.info( 'Unknown error fetching NZB, retry after 2 min %s', url) when = 120 self.add(url, future_nzo, when) # Check if a supported archive else: if dirscanner.ProcessArchiveFile( filename, fn, pp, script, cat, priority=priority, nzbname=nzbname, url=future_nzo.url)[0] == 0: NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) else: # Not a supported filetype, not an nzb (text/html ect) try: os.remove(fn) except: pass logging.info( 'Unknown filetype when fetching NZB, retry after 30s %s', url) self.add(url, future_nzo, 30) except: logging.error('URLGRABBER CRASHED', exc_info=True) logging.debug("URLGRABBER Traceback: ", exc_info=True)
def run(self): logging.info('URLGrabber starting up') self.shutdown = False while not self.shutdown: # Don't pound the website! time.sleep(5.0) (url, future_nzo) = self.queue.get() if not url: # stop signal, go test self.shutdown continue if future_nzo and future_nzo.wait and future_nzo.wait > time.time( ): # Re-queue when too early and still active self.add(url, future_nzo) continue url = url.replace(' ', '') try: if future_nzo: # If nzo entry deleted, give up try: deleted = future_nzo.deleted except AttributeError: deleted = True if deleted: logging.debug('Dropping URL %s, job entry missing', url) continue logging.info('Grabbing URL %s', url) req = urllib2.Request(url) req.add_header('User-Agent', 'SABnzbd+/%s' % sabnzbd.version.__version__) if not [True for item in _BAD_GZ_HOSTS if item in url]: req.add_header('Accept-encoding', 'gzip') filename = None category = None gzipped = False nzo_info = {} wait = 0 retry = True fn = None try: fn = urllib2.urlopen(req) except: # Cannot list exceptions here, because of unpredictability over platforms error0 = str(sys.exc_info()[0]).lower() error1 = str(sys.exc_info()[1]).lower() logging.debug('Error "%s" trying to get the url %s', error1, url) if 'certificate_verify_failed' in error1 or 'certificateerror' in error0: msg = T('Server %s uses an untrusted HTTPS certificate' ) % '' retry = False elif 'nodename nor servname provided' in error1: msg = T('Server name does not resolve') retry = False elif '401' in error1 or 'unauthorized' in error1: msg = T('Unauthorized access') retry = False elif '404' in error1: msg = T('File not on server') retry = False new_url = dereferring(url, fn) if new_url: self.add(new_url, future_nzo) continue if fn: for hdr in fn.headers: try: item = hdr.lower() value = fn.headers[hdr] except: continue if item in ('content-encoding', ) and value == 'gzip': gzipped = True if item in ('category_id', 'x-dnzb-category'): category = value elif item in ('x-dnzb-moreinfo', ): nzo_info['more_info'] = value elif item in ('x-dnzb-name', ): filename = value if not filename.endswith('.nzb'): filename += '.nzb' elif item == 'x-dnzb-propername': nzo_info['propername'] = value elif item == 'x-dnzb-episodename': nzo_info['episodename'] = value elif item == 'x-dnzb-year': nzo_info['year'] = value elif item == 'x-dnzb-failure': nzo_info['failure'] = value elif item == 'x-dnzb-details': nzo_info['details'] = value elif item == 'x-dnzb-password': nzo_info['password'] = value elif item == 'retry-after': # For NZBFinder wait = misc.int_conv(value) if not filename and "filename=" in value: filename = value[value.index("filename=") + 9:].strip(';').strip('"') if wait: # For sites that have a rate-limiting attribute msg = '' retry = True fn = None elif retry: fn, msg, retry, wait, data = _analyse(fn, url) if not fn: if retry: logging.info('Retry URL %s', url) self.add(url, future_nzo, wait) else: bad_fetch(future_nzo, url, msg) continue if not filename: filename = os.path.basename(url) + '.nzb' elif '&nzbname=' in filename: # Sometimes the filename contains the full URL, duh! filename = filename[filename.find('&nzbname=') + 9:] pp = future_nzo.pp script = future_nzo.script cat = future_nzo.cat if (cat is None or cat == '*') and category: cat = misc.cat_convert(category) priority = future_nzo.priority nzbname = future_nzo.custom_name # process data if gzipped: filename = filename + '.gz' if not data: data = fn.read() fn.close() # Sanatize filename first filename = misc.sanitize_filename(filename) # Write data to temp file path = os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER) path = os.path.join(path, filename) f = open(path, 'wb') f.write(data) f.close() del data # Check if nzb file if os.path.splitext(filename)[1].lower() in ('.nzb', '.gz', 'bz2'): res = dirscanner.ProcessSingleFile( filename, path, pp=pp, script=script, cat=cat, priority=priority, nzbname=nzbname, nzo_info=nzo_info, url=future_nzo.url, keep=False, nzo_id=future_nzo.nzo_id)[0] if res: if res == -2: logging.info( 'Incomplete NZB, retry after 5 min %s', url) when = 300 elif res == -1: # Error, but no reason to retry. Warning is already given NzbQueue.do.remove(future_nzo.nzo_id, add_to_history=False) continue else: logging.info( 'Unknown error fetching NZB, retry after 2 min %s', url) when = 120 self.add(url, future_nzo, when) # Check if a supported archive else: if dirscanner.ProcessArchiveFile( filename, path, pp, script, cat, priority=priority, nzbname=nzbname, url=future_nzo.url, keep=False, nzo_id=future_nzo.nzo_id)[0]: # Not a supported filetype, not an nzb (text/html ect) try: os.remove(fn) except: pass logging.info( 'Unknown filetype when fetching NZB, retry after 30s %s', url) self.add(url, future_nzo, 30) except: logging.error(T('URLGRABBER CRASHED'), exc_info=True) logging.debug("URLGRABBER Traceback: ", exc_info=True)