def send_request(
self,
method=None,
url=None,
data=None,
cookies=None,
ignore_errors=False,
follow_redirect=False): # Shared.options['process_timeout']
if not method:
method = self.request.method
if not url:
url = self.request.url
if method in ("POST", "PUT"):
if not data:
data = self.request.data if self.request.data else ""
if not cookies:
cookies = []
jar_request = cookielib.LWPCookieJar()
ret = {
"code": None,
"url": None,
"headers": None,
"body": None,
"time": None
}
while True:
try:
existing_cookies = []
for cookie in self.request.cookies:
clc = cookie.get_cookielib_cookie()
for c in cookies:
if c['name'] == cookie.name:
clc.value = c['value']
existing_cookies.append(c)
jar_request.set_cookie(clc)
for cookie in [
x for x in cookies if x not in existing_cookies
]:
c = Cookie(cookie) # check what to do with cookie.setter
jar_request.set_cookie(c.get_cookielib_cookie())
opener = self.urllib2_opener(self.request, None,
follow_redirect)
req = urllib2.Request(url=url,
data=data.encode() if data else None)
req.get_method = lambda: method
jar_request.add_cookie_header(req)
# headers = self.request.extra_headers
# if self.extra_headers:
# for h in self.extra_headers:
# headers[h] = self.extra_headers[h]
# for hn in headers:
# req.add_header(hn, headers[hn])
if data and not 'Content-type' in req.headers:
req.add_header("Content-type", detect_content_type(data))
now = time.time()
try:
res = opener.open(req, None, self.timeout)
except urllib2.HTTPError as e:
if not ignore_errors:
raise
res = e
opener.close()
ret['code'] = res.getcode()
ret['url'] = res.geturl()
#ret['headers'] = [x.strip() for x in res.info().headers]
ret['headers'] = ["%s: %s" % x for x in res.info().items()]
ret['body'] = res.read()
ret['time'] = time.time() - now
break
except Exception as e:
self.retries -= 1
if self.retries == 0: raise
time.sleep(self.retries_interval)
return ret
def send_request(self,
method=None,
url=None,
data=None,
cookies=None,
ignore_errors=False): # Shared.options['process_timeout']
if not method:
method = self.request.method
if not url:
url = self.request.url
if method == "POST":
if not data:
data = self.request.data if self.request.data else ""
if not cookies:
cookies = []
jar_request = cookielib.LWPCookieJar()
ret = {
"code": None,
"url": None,
"headers": None,
"body": None,
"time": None
}
while True:
try:
existing_cookies = []
for cookie in self.request.cookies:
clc = cookie.get_cookielib_cookie()
for c in cookies:
if c['name'] == cookie.name:
clc.value = c['value']
existing_cookies.append(c)
jar_request.set_cookie(clc)
for cookie in [
x for x in cookies if x not in existing_cookies
]:
c = Cookie(cookie) # check what to do with cookie.setter
jar_request.set_cookie(c.get_cookielib_cookie())
opener = self.urllib2_opener(self.request, None, True)
req = urllib2.Request(url=url, data=data)
jar_request.add_cookie_header(req)
if self.extra_headers:
for hn in self.extra_headers:
req.add_header(hn, self.extra_headers[hn])
now = time.time()
try:
res = opener.open(req, None, self.timeout)
except urllib2.HTTPError as e:
if not ignore_errors:
raise
res = e
opener.close()
ret['code'] = res.getcode()
ret['url'] = res.geturl()
ret['headers'] = [x.strip() for x in res.info().headers]
ret['body'] = res.read()
ret['time'] = time.time() - now
break
except Exception as e:
self.retries -= 1
if self.retries == 0: raise
time.sleep(self.retries_interval)
return ret