def __init__(self,
url,
method='GET',
data=None,
headers=None,
headers_only=False,
user_agent=None,
follow_location=False,
force_quiet=True):
GObjectWrapper.__init__(self)
self.result = StringIO.StringIO()
self.result_headers = StringIO.StringIO()
if isinstance(url, unicode):
self.url = url.encode("utf-8")
else:
self.url = url
self.method = method
self.data = data
self.headers = headers
self.status = None
# the actual curl request object
self.curl = pycurl.Curl()
if (logging.root.level == logging.DEBUG and not force_quiet):
self.curl.setopt(pycurl.VERBOSE, 1)
self.curl.setopt(pycurl.WRITEFUNCTION, self.result.write)
self.curl.setopt(pycurl.HEADERFUNCTION, self.result_headers.write)
# We want to use gzip and deflate if possible:
self.curl.setopt(pycurl.ENCODING, "") # use all available encodings
self.curl.setopt(pycurl.URL, self.url)
# let's set the HTTP request method
if method == 'GET':
self.curl.setopt(pycurl.HTTPGET, 1)
elif method == 'POST':
self.curl.setopt(pycurl.POST, 1)
elif method == 'PUT':
self.curl.setopt(pycurl.UPLOAD, 1)
else:
self.curl.setopt(pycurl.CUSTOMREQUEST, method)
if data:
if method == "PUT":
self.data = StringIO.StringIO(data)
self.curl.setopt(pycurl.READFUNCTION, self.data.read)
self.curl.setopt(pycurl.INFILESIZE, len(self.data.getvalue()))
else:
self.curl.setopt(pycurl.POSTFIELDS, self.data)
self.curl.setopt(pycurl.POSTFIELDSIZE, len(self.data))
if headers:
self.curl.setopt(pycurl.HTTPHEADER, headers)
if headers_only:
self.curl.setopt(pycurl.HEADER, 1)
self.curl.setopt(pycurl.NOBODY, 1)
if user_agent:
self.curl.setopt(pycurl.USERAGENT, user_agent)
if follow_location:
self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
# Proxy: let's be careful to isolate the protocol to ensure that we
# support the case where http and https might use different proxies
split_url = self.url.split('://', 1)
if len(split_url) > 1:
#We were able to get a protocol
protocol = split_url[0]
proxies = urllib.getproxies()
if protocol in proxies:
self.curl.setopt(pycurl.PROXY, proxies[protocol])
# On Windows urllib.getproxies() doesn't contain https if "Use the
# same proxy for all protocols" is selected. So we might want to
# guess that the http proxy is useful, but we have no way to know
# if the https proxy is intentionally not specified. Environment
# variables and separately specified (even if identical) settings
# work as expected.
# Possible code to reuse the http proxy for https:
# elif protocol is 'https' and 'http' in proxies:
# self.curl.setopt(pycurl.PROXY, proxies['http'])
# self reference required, because CurlMulti will only return
# Curl handles
self.curl.request = self
def __init__(self, url, method='GET', data=None, headers=None,
headers_only=False, user_agent=None, follow_location=False,
force_quiet=True):
GObjectWrapper.__init__(self)
self.result = StringIO.StringIO()
self.result_headers = StringIO.StringIO()
if isinstance(url, unicode):
self.url = url.encode("utf-8")
else:
self.url = url
self.method = method
self.data = data
self.headers = headers
self.status = None
# the actual curl request object
self.curl = pycurl.Curl()
if (logging.root.level == logging.DEBUG and not force_quiet):
self.curl.setopt(pycurl.VERBOSE, 1)
self.curl.setopt(pycurl.WRITEFUNCTION, self.result.write)
self.curl.setopt(pycurl.HEADERFUNCTION, self.result_headers.write)
# We want to use gzip and deflate if possible:
self.curl.setopt(pycurl.ENCODING, "") # use all available encodings
self.curl.setopt(pycurl.URL, self.url)
# let's set the HTTP request method
if method == 'GET':
self.curl.setopt(pycurl.HTTPGET, 1)
elif method == 'POST':
self.curl.setopt(pycurl.POST, 1)
elif method == 'PUT':
self.curl.setopt(pycurl.UPLOAD, 1)
else:
self.curl.setopt(pycurl.CUSTOMREQUEST, method)
if data:
if method == "PUT":
self.data = StringIO.StringIO(data)
self.curl.setopt(pycurl.READFUNCTION, self.data.read)
self.curl.setopt(pycurl.INFILESIZE, len(self.data.getvalue()))
else:
self.curl.setopt(pycurl.POSTFIELDS, self.data)
self.curl.setopt(pycurl.POSTFIELDSIZE, len(self.data))
if headers:
self.curl.setopt(pycurl.HTTPHEADER, headers)
if headers_only:
self.curl.setopt(pycurl.HEADER, 1)
self.curl.setopt(pycurl.NOBODY, 1)
if user_agent:
self.curl.setopt(pycurl.USERAGENT, user_agent)
if follow_location:
self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
if libproxy:
for proxy in proxy_factory.getProxies(self.url):
# if we connect to localhost (localtm) with proxy specifically
# set to direct://, libcurl connects fine, but then asks
# GET http://localhost:55555/unit/en/af/whatever
# instead of
# GET /unit/en/af/whatever
# and it doesn't work. We have to set it specifically to ""
# though, otherwise it seems to fall back to environment
# variables.
if proxy == "direct://":
proxy = ""
self.curl.setopt(pycurl.PROXY, proxy)
#only use the first one
break
else:
# Proxy: let's be careful to isolate the protocol to ensure that we
# support the case where http and https might use different proxies
split_url = self.url.split('://', 1)
if len(split_url) > 1:
#We were able to get a protocol
protocol, address = split_url
host, _path = urllib.splithost('//' + address)
proxies = urllib.getproxies()
if protocol in proxies and not urllib.proxy_bypass(host):
self.curl.setopt(pycurl.PROXY, proxies[protocol])
# self reference required, because CurlMulti will only return
# Curl handles
self.curl.request = self
def __init__(self, url, method='GET', data=None, headers=None,
headers_only=False, user_agent=None, follow_location=False,
force_quiet=True):
GObjectWrapper.__init__(self)
self.result = StringIO.StringIO()
self.result_headers = StringIO.StringIO()
if isinstance(url, unicode):
self.url = url.encode("utf-8")
else:
self.url = url
self.method = method
self.data = data
self.headers = headers
self.status = None
# the actual curl request object
self.curl = pycurl.Curl()
if (logging.root.level == logging.DEBUG and not force_quiet):
self.curl.setopt(pycurl.VERBOSE, 1)
self.curl.setopt(pycurl.WRITEFUNCTION, self.result.write)
self.curl.setopt(pycurl.HEADERFUNCTION, self.result_headers.write)
# We want to use gzip and deflate if possible:
self.curl.setopt(pycurl.ENCODING, "") # use all available encodings
self.curl.setopt(pycurl.URL, self.url)
# let's set the HTTP request method
if method == 'GET':
self.curl.setopt(pycurl.HTTPGET, 1)
elif method == 'POST':
self.curl.setopt(pycurl.POST, 1)
elif method == 'PUT':
self.curl.setopt(pycurl.UPLOAD, 1)
else:
self.curl.setopt(pycurl.CUSTOMREQUEST, method)
if data:
if method == "PUT":
self.data = StringIO.StringIO(data)
self.curl.setopt(pycurl.READFUNCTION, self.data.read)
self.curl.setopt(pycurl.INFILESIZE, len(self.data.getvalue()))
else:
self.curl.setopt(pycurl.POSTFIELDS, self.data)
self.curl.setopt(pycurl.POSTFIELDSIZE, len(self.data))
if headers:
self.curl.setopt(pycurl.HTTPHEADER, headers)
if headers_only:
self.curl.setopt(pycurl.HEADER, 1)
self.curl.setopt(pycurl.NOBODY, 1)
if user_agent:
self.curl.setopt(pycurl.USERAGENT, user_agent)
if follow_location:
self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
# Proxy: let's be careful to isolate the protocol to ensure that we
# support the case where http and https might use different proxies
split_url = self.url.split('://', 1)
if len(split_url) > 1:
#We were able to get a protocol
protocol = split_url[0]
proxies = urllib.getproxies()
if protocol in proxies:
self.curl.setopt(pycurl.PROXY, proxies[protocol])
# On Windows urllib.getproxies() doesn't contain https if "Use the
# same proxy for all protocols" is selected. So we might want to
# guess that the http proxy is useful, but we have no way to know
# if the https proxy is intentionally not specified. Environment
# variables and separately specified (even if identical) settings
# work as expected.
# Possible code to reuse the http proxy for https:
# elif protocol is 'https' and 'http' in proxies:
# self.curl.setopt(pycurl.PROXY, proxies['http'])
# self reference required, because CurlMulti will only return
# Curl handles
self.curl.request = self
def __init__(
self,
url,
method="GET",
data=None,
headers=None,
headers_only=False,
user_agent=None,
follow_location=False,
force_quiet=True,
):
GObjectWrapper.__init__(self)
self.result = StringIO.StringIO()
self.result_headers = StringIO.StringIO()
if isinstance(url, unicode):
self.url = url.encode("utf-8")
else:
self.url = url
self.method = method
self.data = data
self.headers = headers
self.status = None
# the actual curl request object
self.curl = pycurl.Curl()
if logging.root.level == logging.DEBUG and not force_quiet:
self.curl.setopt(pycurl.VERBOSE, 1)
self.curl.setopt(pycurl.WRITEFUNCTION, self.result.write)
self.curl.setopt(pycurl.HEADERFUNCTION, self.result_headers.write)
# We want to use gzip and deflate if possible:
self.curl.setopt(pycurl.ENCODING, "") # use all available encodings
self.curl.setopt(pycurl.URL, self.url)
# let's set the HTTP request method
if method == "GET":
self.curl.setopt(pycurl.HTTPGET, 1)
elif method == "POST":
self.curl.setopt(pycurl.POST, 1)
elif method == "PUT":
self.curl.setopt(pycurl.UPLOAD, 1)
else:
self.curl.setopt(pycurl.CUSTOMREQUEST, method)
if data:
if method == "PUT":
self.data = StringIO.StringIO(data)
self.curl.setopt(pycurl.READFUNCTION, self.data.read)
self.curl.setopt(pycurl.INFILESIZE, len(self.data.getvalue()))
else:
self.curl.setopt(pycurl.POSTFIELDS, self.data)
self.curl.setopt(pycurl.POSTFIELDSIZE, len(self.data))
if headers:
self.curl.setopt(pycurl.HTTPHEADER, headers)
if headers_only:
self.curl.setopt(pycurl.HEADER, 1)
self.curl.setopt(pycurl.NOBODY, 1)
if user_agent:
self.curl.setopt(pycurl.USERAGENT, user_agent)
if follow_location:
self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
if libproxy:
for proxy in proxy_factory.getProxies(self.url):
# only use the first one
self.curl.setopt(pycurl.PROXY, proxy)
break
else:
# Proxy: let's be careful to isolate the protocol to ensure that we
# support the case where http and https might use different proxies
split_url = self.url.split("://", 1)
if len(split_url) > 1:
# We were able to get a protocol
protocol, address = split_url
host, _path = urllib.splithost("//" + address)
proxies = urllib.getproxies()
if protocol in proxies and not urllib.proxy_bypass(host):
self.curl.setopt(pycurl.PROXY, proxies[protocol])
# self reference required, because CurlMulti will only return
# Curl handles
self.curl.request = self
