def send_request( self, method=None, url=None, data=None, cookies=None, ignore_errors=False, follow_redirect=False): # Shared.options['process_timeout'] if not method: method = self.request.method if not url: url = self.request.url if method in ("POST", "PUT"): if not data: data = self.request.data if self.request.data else "" if not cookies: cookies = [] jar_request = cookielib.LWPCookieJar() ret = { "code": None, "url": None, "headers": None, "body": None, "time": None } while True: try: existing_cookies = [] for cookie in self.request.cookies: clc = cookie.get_cookielib_cookie() for c in cookies: if c['name'] == cookie.name: clc.value = c['value'] existing_cookies.append(c) jar_request.set_cookie(clc) for cookie in [ x for x in cookies if x not in existing_cookies ]: c = Cookie(cookie) # check what to do with cookie.setter jar_request.set_cookie(c.get_cookielib_cookie()) opener = self.urllib2_opener(self.request, None, follow_redirect) req = urllib2.Request(url=url, data=data.encode() if data else None) req.get_method = lambda: method jar_request.add_cookie_header(req) # headers = self.request.extra_headers # if self.extra_headers: # for h in self.extra_headers: # headers[h] = self.extra_headers[h] # for hn in headers: # req.add_header(hn, headers[hn]) if data and not 'Content-type' in req.headers: req.add_header("Content-type", detect_content_type(data)) now = time.time() try: res = opener.open(req, None, self.timeout) except urllib2.HTTPError as e: if not ignore_errors: raise res = e opener.close() ret['code'] = res.getcode() ret['url'] = res.geturl() #ret['headers'] = [x.strip() for x in res.info().headers] ret['headers'] = ["%s: %s" % x for x in res.info().items()] ret['body'] = res.read() ret['time'] = time.time() - now break except Exception as e: self.retries -= 1 if self.retries == 0: raise time.sleep(self.retries_interval) return ret
def send_request(self, method=None, url=None, data=None, cookies=None, ignore_errors=False): # Shared.options['process_timeout'] if not method: method = self.request.method if not url: url = self.request.url if method == "POST": if not data: data = self.request.data if self.request.data else "" if not cookies: cookies = [] jar_request = cookielib.LWPCookieJar() ret = { "code": None, "url": None, "headers": None, "body": None, "time": None } while True: try: existing_cookies = [] for cookie in self.request.cookies: clc = cookie.get_cookielib_cookie() for c in cookies: if c['name'] == cookie.name: clc.value = c['value'] existing_cookies.append(c) jar_request.set_cookie(clc) for cookie in [ x for x in cookies if x not in existing_cookies ]: c = Cookie(cookie) # check what to do with cookie.setter jar_request.set_cookie(c.get_cookielib_cookie()) opener = self.urllib2_opener(self.request, None, True) req = urllib2.Request(url=url, data=data) jar_request.add_cookie_header(req) if self.extra_headers: for hn in self.extra_headers: req.add_header(hn, self.extra_headers[hn]) now = time.time() try: res = opener.open(req, None, self.timeout) except urllib2.HTTPError as e: if not ignore_errors: raise res = e opener.close() ret['code'] = res.getcode() ret['url'] = res.geturl() ret['headers'] = [x.strip() for x in res.info().headers] ret['body'] = res.read() ret['time'] = time.time() - now break except Exception as e: self.retries -= 1 if self.retries == 0: raise time.sleep(self.retries_interval) return ret