def __init__(self, url, method='GET', data=None, headers=None, headers_only=False, user_agent=None, follow_location=False, force_quiet=True): GObjectWrapper.__init__(self) self.result = StringIO.StringIO() self.result_headers = StringIO.StringIO() if isinstance(url, unicode): self.url = url.encode("utf-8") else: self.url = url self.method = method self.data = data self.headers = headers self.status = None # the actual curl request object self.curl = pycurl.Curl() if (logging.root.level == logging.DEBUG and not force_quiet): self.curl.setopt(pycurl.VERBOSE, 1) self.curl.setopt(pycurl.WRITEFUNCTION, self.result.write) self.curl.setopt(pycurl.HEADERFUNCTION, self.result_headers.write) # We want to use gzip and deflate if possible: self.curl.setopt(pycurl.ENCODING, "") # use all available encodings self.curl.setopt(pycurl.URL, self.url) # let's set the HTTP request method if method == 'GET': self.curl.setopt(pycurl.HTTPGET, 1) elif method == 'POST': self.curl.setopt(pycurl.POST, 1) elif method == 'PUT': self.curl.setopt(pycurl.UPLOAD, 1) else: self.curl.setopt(pycurl.CUSTOMREQUEST, method) if data: if method == "PUT": self.data = StringIO.StringIO(data) self.curl.setopt(pycurl.READFUNCTION, self.data.read) self.curl.setopt(pycurl.INFILESIZE, len(self.data.getvalue())) else: self.curl.setopt(pycurl.POSTFIELDS, self.data) self.curl.setopt(pycurl.POSTFIELDSIZE, len(self.data)) if headers: self.curl.setopt(pycurl.HTTPHEADER, headers) if headers_only: self.curl.setopt(pycurl.HEADER, 1) self.curl.setopt(pycurl.NOBODY, 1) if user_agent: self.curl.setopt(pycurl.USERAGENT, user_agent) if follow_location: self.curl.setopt(pycurl.FOLLOWLOCATION, 1) # Proxy: let's be careful to isolate the protocol to ensure that we # support the case where http and https might use different proxies split_url = self.url.split('://', 1) if len(split_url) > 1: #We were able to get a protocol protocol = split_url[0] proxies = urllib.getproxies() if protocol in proxies: self.curl.setopt(pycurl.PROXY, proxies[protocol]) # On Windows urllib.getproxies() doesn't contain https if "Use the # same proxy for all protocols" is selected. So we might want to # guess that the http proxy is useful, but we have no way to know # if the https proxy is intentionally not specified. Environment # variables and separately specified (even if identical) settings # work as expected. # Possible code to reuse the http proxy for https: # elif protocol is 'https' and 'http' in proxies: # self.curl.setopt(pycurl.PROXY, proxies['http']) # self reference required, because CurlMulti will only return # Curl handles self.curl.request = self
def __init__(self, url, method='GET', data=None, headers=None, headers_only=False, user_agent=None, follow_location=False, force_quiet=True): GObjectWrapper.__init__(self) self.result = StringIO.StringIO() self.result_headers = StringIO.StringIO() if isinstance(url, unicode): self.url = url.encode("utf-8") else: self.url = url self.method = method self.data = data self.headers = headers self.status = None # the actual curl request object self.curl = pycurl.Curl() if (logging.root.level == logging.DEBUG and not force_quiet): self.curl.setopt(pycurl.VERBOSE, 1) self.curl.setopt(pycurl.WRITEFUNCTION, self.result.write) self.curl.setopt(pycurl.HEADERFUNCTION, self.result_headers.write) # We want to use gzip and deflate if possible: self.curl.setopt(pycurl.ENCODING, "") # use all available encodings self.curl.setopt(pycurl.URL, self.url) # let's set the HTTP request method if method == 'GET': self.curl.setopt(pycurl.HTTPGET, 1) elif method == 'POST': self.curl.setopt(pycurl.POST, 1) elif method == 'PUT': self.curl.setopt(pycurl.UPLOAD, 1) else: self.curl.setopt(pycurl.CUSTOMREQUEST, method) if data: if method == "PUT": self.data = StringIO.StringIO(data) self.curl.setopt(pycurl.READFUNCTION, self.data.read) self.curl.setopt(pycurl.INFILESIZE, len(self.data.getvalue())) else: self.curl.setopt(pycurl.POSTFIELDS, self.data) self.curl.setopt(pycurl.POSTFIELDSIZE, len(self.data)) if headers: self.curl.setopt(pycurl.HTTPHEADER, headers) if headers_only: self.curl.setopt(pycurl.HEADER, 1) self.curl.setopt(pycurl.NOBODY, 1) if user_agent: self.curl.setopt(pycurl.USERAGENT, user_agent) if follow_location: self.curl.setopt(pycurl.FOLLOWLOCATION, 1) if libproxy: for proxy in proxy_factory.getProxies(self.url): # if we connect to localhost (localtm) with proxy specifically # set to direct://, libcurl connects fine, but then asks # GET http://localhost:55555/unit/en/af/whatever # instead of # GET /unit/en/af/whatever # and it doesn't work. We have to set it specifically to "" # though, otherwise it seems to fall back to environment # variables. if proxy == "direct://": proxy = "" self.curl.setopt(pycurl.PROXY, proxy) #only use the first one break else: # Proxy: let's be careful to isolate the protocol to ensure that we # support the case where http and https might use different proxies split_url = self.url.split('://', 1) if len(split_url) > 1: #We were able to get a protocol protocol, address = split_url host, _path = urllib.splithost('//' + address) proxies = urllib.getproxies() if protocol in proxies and not urllib.proxy_bypass(host): self.curl.setopt(pycurl.PROXY, proxies[protocol]) # self reference required, because CurlMulti will only return # Curl handles self.curl.request = self
def __init__( self, url, method="GET", data=None, headers=None, headers_only=False, user_agent=None, follow_location=False, force_quiet=True, ): GObjectWrapper.__init__(self) self.result = StringIO.StringIO() self.result_headers = StringIO.StringIO() if isinstance(url, unicode): self.url = url.encode("utf-8") else: self.url = url self.method = method self.data = data self.headers = headers self.status = None # the actual curl request object self.curl = pycurl.Curl() if logging.root.level == logging.DEBUG and not force_quiet: self.curl.setopt(pycurl.VERBOSE, 1) self.curl.setopt(pycurl.WRITEFUNCTION, self.result.write) self.curl.setopt(pycurl.HEADERFUNCTION, self.result_headers.write) # We want to use gzip and deflate if possible: self.curl.setopt(pycurl.ENCODING, "") # use all available encodings self.curl.setopt(pycurl.URL, self.url) # let's set the HTTP request method if method == "GET": self.curl.setopt(pycurl.HTTPGET, 1) elif method == "POST": self.curl.setopt(pycurl.POST, 1) elif method == "PUT": self.curl.setopt(pycurl.UPLOAD, 1) else: self.curl.setopt(pycurl.CUSTOMREQUEST, method) if data: if method == "PUT": self.data = StringIO.StringIO(data) self.curl.setopt(pycurl.READFUNCTION, self.data.read) self.curl.setopt(pycurl.INFILESIZE, len(self.data.getvalue())) else: self.curl.setopt(pycurl.POSTFIELDS, self.data) self.curl.setopt(pycurl.POSTFIELDSIZE, len(self.data)) if headers: self.curl.setopt(pycurl.HTTPHEADER, headers) if headers_only: self.curl.setopt(pycurl.HEADER, 1) self.curl.setopt(pycurl.NOBODY, 1) if user_agent: self.curl.setopt(pycurl.USERAGENT, user_agent) if follow_location: self.curl.setopt(pycurl.FOLLOWLOCATION, 1) if libproxy: for proxy in proxy_factory.getProxies(self.url): # only use the first one self.curl.setopt(pycurl.PROXY, proxy) break else: # Proxy: let's be careful to isolate the protocol to ensure that we # support the case where http and https might use different proxies split_url = self.url.split("://", 1) if len(split_url) > 1: # We were able to get a protocol protocol, address = split_url host, _path = urllib.splithost("//" + address) proxies = urllib.getproxies() if protocol in proxies and not urllib.proxy_bypass(host): self.curl.setopt(pycurl.PROXY, proxies[protocol]) # self reference required, because CurlMulti will only return # Curl handles self.curl.request = self