def from_request(cls, http_method, http_url, headers=None, parameters=None, query_string=None): """Combines multiple parameter sources.""" if parameters is None: parameters = {} # Headers if headers and 'Authorization' in headers: auth_header = headers['Authorization'] # Check that the authorization header is OAuth. if auth_header[:6] == 'OAuth ': auth_header = auth_header[6:] try: # Get the parameters from the header. header_params = cls._split_header(auth_header) parameters.update(header_params) except Exception: raise Error('Unable to parse OAuth parameters from Authorization header.') # GET or POST query string. if query_string: query_params = cls._split_url_string(query_string) parameters.update(query_params) # URL parameters. param_str = urlparse(http_url)[4] # query url_params = cls._split_url_string(param_str) parameters.update(url_params) if parameters: return cls(http_method, http_url, parameters) return None
def get_callback_url(self): if self.callback and self.verifier: # Append the oauth_verifier. parts = urlparse(self.callback) scheme, netloc, path, params, query, fragment = parts[:6] if query: query = '%s&oauth_verifier=%s' % (query, self.verifier) else: query = 'oauth_verifier=%s' % self.verifier return urlunparse((scheme, netloc, path, params, query, fragment)) return self.callback
def redirect_url(genhost, url): """ libgen.io might have dns blocked, but user can bypass using genhost 93.174.95.27 in config libgen might send us a book url that still contains http://libgen.io/ or /libgen.io/ so we might need to redirect it to users genhost setting """ myurl = urlparse(url) if myurl.netloc.lower() != 'libgen.io': return url host = urlparse(genhost) # genhost http://93.174.95.27 -> scheme http, netloc 93.174.95.27, path "" # genhost 93.174.95.27 -> scheme "", netloc "", path 93.174.95.27 if host.netloc: if host.netloc.lower() != 'libgen.io': # noinspection PyArgumentList myurl = myurl._replace(**{"netloc": host.netloc}) logger.debug('Redirected libgen.io to [%s]' % host.netloc) elif host.path: if host.path.lower() != 'libgen.io': # noinspection PyArgumentList myurl = myurl._replace(**{"netloc": host.netloc}) logger.debug('Redirected libgen.io to [%s]' % host.netloc) return myurl.geturl()
def redirect_url(genhost, url): """ libgen.io might have dns blocked, but user can bypass using genhost 93.174.95.27 in config libgen might send us a book url that still contains http://libgen.io/ or /libgen.io/ so we might need to redirect it to users genhost setting """ myurl = urlparse(url) if myurl.netloc.lower() != 'libgen.io': return url host = urlparse(genhost) # genhost http://93.174.95.27 -> scheme http, netloc 93.174.95.27, path "" # genhost 93.174.95.27 -> scheme "", netloc "", path 93.174.95.27 if host.netloc: if host.netloc.lower() != 'libgen.io': # noinspection PyArgumentList myurl = myurl._replace(**{"netloc": host.netloc}) logger.debug('Redirected libgen.io to [%s]' % host.netloc) elif host.path: if host.path.lower() != 'libgen.io': # noinspection PyArgumentList myurl = myurl._replace(**{"netloc": host.netloc}) logger.debug('Redirected libgen.io to [%s]' % host.netloc) return myurl.geturl()
def allowed_token(self, token): if "data" in token: attrs = token["data"] attr_names = set(attrs.keys()) # Remove forbidden attributes for to_remove in (attr_names - self.allowed_attributes): del token["data"][to_remove] attr_names.remove(to_remove) # Remove attributes with disallowed URL values for attr in (attr_names & self.attr_val_is_uri): assert attr in attrs # I don't have a clue where this regexp comes from or why it matches those # characters, nor why we call unescape. I just know it's always been here. # Should you be worried by this comment in a sanitizer? Yes. On the other hand, all # this will do is remove *more* than it otherwise would. val_unescaped = re.sub("[`\x00-\x20\x7f-\xa0\s]+", '', unescape(attrs[attr])).lower() # remove replacement characters from unescaped characters val_unescaped = val_unescaped.replace("\ufffd", "") try: uri = urlparse.urlparse(val_unescaped) except ValueError: uri = None del attrs[attr] if uri and uri.scheme: if uri.scheme not in self.allowed_protocols: del attrs[attr] if uri.scheme == 'data': m = data_content_type.match(uri.path) if not m: del attrs[attr] elif m.group('content_type' ) not in self.allowed_content_types: del attrs[attr] for attr in self.svg_attr_val_allows_ref: if attr in attrs: attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', ' ', unescape(attrs[attr])) if (token["name"] in self.svg_allow_local_href and (namespaces['xlink'], 'href') in attrs and re.search( '^\s*[^#\s].*', attrs[(namespaces['xlink'], 'href')])): del attrs[(namespaces['xlink'], 'href')] if (None, 'style') in attrs: attrs[(None, 'style')] = self.sanitize_css(attrs[(None, 'style')]) token["data"] = attrs return token
def allowed_token(self, token): if "data" in token: attrs = token["data"] attr_names = set(attrs.keys()) # Remove forbidden attributes for to_remove in (attr_names - self.allowed_attributes): del token["data"][to_remove] attr_names.remove(to_remove) # Remove attributes with disallowed URL values for attr in (attr_names & self.attr_val_is_uri): assert attr in attrs # I don't have a clue where this regexp comes from or why it matches those # characters, nor why we call unescape. I just know it's always been here. # Should you be worried by this comment in a sanitizer? Yes. On the other hand, all # this will do is remove *more* than it otherwise would. val_unescaped = re.sub("[`\x00-\x20\x7f-\xa0\s]+", '', unescape(attrs[attr])).lower() # remove replacement characters from unescaped characters val_unescaped = val_unescaped.replace("\ufffd", "") try: uri = urlparse.urlparse(val_unescaped) except ValueError: uri = None del attrs[attr] if uri and uri.scheme: if uri.scheme not in self.allowed_protocols: del attrs[attr] if uri.scheme == 'data': m = data_content_type.match(uri.path) if not m: del attrs[attr] elif m.group('content_type') not in self.allowed_content_types: del attrs[attr] for attr in self.svg_attr_val_allows_ref: if attr in attrs: attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)', ' ', unescape(attrs[attr])) if (token["name"] in self.svg_allow_local_href and (namespaces['xlink'], 'href') in attrs and re.search('^\s*[^#\s].*', attrs[(namespaces['xlink'], 'href')])): del attrs[(namespaces['xlink'], 'href')] if (None, 'style') in attrs: attrs[(None, 'style')] = self.sanitize_css(attrs[(None, 'style')]) token["data"] = attrs return token
def url(self, value): parts = urlparse(value) scheme, netloc, path = parts[:3] # Exclude default port numbers. if scheme == 'http' and netloc[-3:] == ':80': netloc = netloc[:-3] elif scheme == 'https' and netloc[-4:] == ':443': netloc = netloc[:-4] if scheme != 'http' and scheme != 'https': raise ValueError("Unsupported URL %s (%s)." % (value, scheme)) value = '%s://%s%s' % (scheme, netloc, path) self.__dict__['url'] = value
def request(self, uri, method="GET", body=None, headers=None, redirections=httplib2.DEFAULT_MAX_REDIRECTS, connection_type=None, force_auth_header=False): if not isinstance(headers, dict): headers = {} if body and method == "POST": parameters = dict(parse_qsl(body)) elif method == "GET": parsed = urlparse(uri) parameters = parse_qs(parsed.query) else: parameters = None req = Request.from_consumer_and_token(self.consumer, token=self.token, http_method=method, http_url=uri, parameters=parameters) req.sign_request(self.method, self.consumer, self.token) if force_auth_header: # ensure we always send Authorization headers.update(req.to_header()) if method == "POST": if not force_auth_header: body = req.to_postdata() else: body = req.encode_postdata(req.get_nonoauth_parameters()) headers['Content-Type'] = 'application/x-www-form-urlencoded' elif method == "GET": if not force_auth_header: uri = req.to_url() else: if not force_auth_header: # don't call update twice. headers.update(req.to_header()) return httplib2.Http.request(self, uri, method=method, body=body, headers=headers, redirections=redirections, connection_type=connection_type)
def torrentAction(method, arguments): global session_id, host_url username = lazylibrarian.CONFIG['TRANSMISSION_USER'] password = lazylibrarian.CONFIG['TRANSMISSION_PASS'] if host_url: if lazylibrarian.LOGLEVEL & lazylibrarian.log_dlcomms: logger.debug("Using existing host %s" % host_url) else: host = lazylibrarian.CONFIG['TRANSMISSION_HOST'] port = check_int(lazylibrarian.CONFIG['TRANSMISSION_PORT'], 0) if not host or not port: logger.error( 'Invalid transmission host or port, check your config') return False if not host.startswith("http://") and not host.startswith("https://"): host = 'http://' + host if host.endswith('/'): host = host[:-1] # Fix the URL. We assume that the user does not point to the RPC endpoint, # so add it if it is missing. parts = list(urlparse(host)) if parts[0] not in ("http", "https"): parts[0] = "http" if ':' not in parts[1]: parts[1] += ":%s" % port if not parts[2].endswith("/rpc"): parts[2] += "/transmission/rpc" host_url = urlunparse(parts) auth = (username, password) if username and password else None proxies = proxyList() timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30) # Retrieve session id if session_id: if lazylibrarian.LOGLEVEL & lazylibrarian.log_dlcomms: logger.debug('Using existing session_id %s' % session_id) else: response = requests.get(host_url, auth=auth, proxies=proxies, timeout=timeout) if response is None: logger.error("Error getting Transmission session ID") return # Parse response if response.status_code == 401: if auth: logger.error( "Username and/or password not accepted by Transmission") else: logger.error("Transmission authorization required") return elif response.status_code == 409: session_id = response.headers['x-transmission-session-id'] if not session_id: logger.error("Expected a Session ID from Transmission, got %s" % response.status_code) return # Prepare next request headers = {'x-transmission-session-id': session_id} data = {'method': method, 'arguments': arguments} try: response = requests.post(host_url, json=data, headers=headers, proxies=proxies, auth=auth, timeout=timeout) if response.status_code == 409: session_id = response.headers['x-transmission-session-id'] logger.debug("Retrying with new session_id %s" % session_id) headers = {'x-transmission-session-id': session_id} response = requests.post(host_url, json=data, headers=headers, proxies=proxies, auth=auth, timeout=timeout) if not str(response.status_code).startswith('2'): logger.error("Expected a response from Transmission, got %s" % response.status_code) return try: res = response.json() except ValueError: logger.error("Expected json, Transmission returned %s" % response.text) res = '' return res except Exception as e: logger.error('Transmission %s: %s' % (type(e).__name__, str(e))) return
def torrentAction(method, arguments): global session_id, host_url username = lazylibrarian.CONFIG['TRANSMISSION_USER'] password = lazylibrarian.CONFIG['TRANSMISSION_PASS'] if host_url: if lazylibrarian.LOGLEVEL & lazylibrarian.log_dlcomms: logger.debug("Using existing host %s" % host_url) else: host = lazylibrarian.CONFIG['TRANSMISSION_HOST'] port = check_int(lazylibrarian.CONFIG['TRANSMISSION_PORT'], 0) if not host or not port: res = 'Invalid transmission host or port, check your config' logger.error(res) return False, res if not host.startswith("http://") and not host.startswith("https://"): host = 'http://' + host if host.endswith('/'): host = host[:-1] # Fix the URL. We assume that the user does not point to the RPC endpoint, # so add it if it is missing. parts = list(urlparse(host)) if parts[0] not in ("http", "https"): parts[0] = "http" if ':' not in parts[1]: parts[1] += ":%s" % port if not parts[2].endswith("/rpc"): parts[2] += "/transmission/rpc" host_url = urlunparse(parts) auth = (username, password) if username and password else None proxies = proxyList() timeout = check_int(lazylibrarian.CONFIG['HTTP_TIMEOUT'], 30) # Retrieve session id if session_id: if lazylibrarian.LOGLEVEL & lazylibrarian.log_dlcomms: logger.debug('Using existing session_id %s' % session_id) else: response = requests.get(host_url, auth=auth, proxies=proxies, timeout=timeout) if response is None: res = "Error getting Transmission session ID" logger.error(res) return False, res # Parse response if response.status_code == 401: if auth: res = "Username and/or password not accepted by Transmission" else: res = "Transmission authorization required" logger.error(res) return False, res elif response.status_code == 409: session_id = response.headers['x-transmission-session-id'] if not session_id: res = "Expected a Session ID from Transmission, got %s" % response.status_code logger.error(res) return False, res # Prepare next request headers = {'x-transmission-session-id': session_id} data = {'method': method, 'arguments': arguments} try: response = requests.post(host_url, json=data, headers=headers, proxies=proxies, auth=auth, timeout=timeout) if response.status_code == 409: session_id = response.headers['x-transmission-session-id'] logger.debug("Retrying with new session_id %s" % session_id) headers = {'x-transmission-session-id': session_id} response = requests.post(host_url, json=data, headers=headers, proxies=proxies, auth=auth, timeout=timeout) if not str(response.status_code).startswith('2'): res = "Expected a response from Transmission, got %s" % response.status_code logger.error(res) return False, res try: res = response.json() except ValueError: res = "Expected json, Transmission returned %s" % response.text logger.error(res) return False, res return res, '' except Exception as e: res = 'Transmission %s: %s' % (type(e).__name__, str(e)) logger.error(res) return False, res