def connection(self, url, payload=""): time.sleep(conf.delay_time) ret = AttribDict() ret.code = 0 ret.text = '' ret.success = False if conf.raw: raw_data = self.raw_request start_time = time.time() status_code, head, content, redirect, log = self.send_raw_request( url, raw_data, payload) end_time = time.time() self.elapsed_time = end_time - start_time ret.code = status_code con = content con = format_unicode(con) ret.text = get_text_value(con) ret.success = True else: if self.method == 'get': payload = url_encode(payload) pass try: url, params, data = self.generate_params(url, payload) if data: data = dict_to_q_str(data) if self.method == 'get': start_time = time.time() status_code, head, content, redirect, log = fly_req.http( url, timeout=self.timeout, header=self.headers, proxy=self.proxies) end_time = time.time() self.elapsed_time = end_time - start_time ret.code = status_code con = content con = format_unicode(con) ret.text = get_text_value(con) ret.success = True if self.method == 'post': start_time = time.time() status_code, head, content, redirect, log = fly_req.http( url + '?' + params, data=data, timeout=self.timeout, header=self.headers, proxy=self.proxies) end_time = time.time() self.elapsed_time = end_time - start_time ret.code = status_code con = content con = format_unicode(con) ret.text = get_text_value(con) ret.success = True except Exception as e: logger.warning(e.message) sys.exit(0) return ret
def getResponseHeaders(url, Agreement="HTTP/1.0", mode="GET"): #返回response中headers的排序list和code (均小写) try: parse = urlparse(url) url = url if parse.scheme != "" else "http://" + url if parse.scheme == "http": http = socket.socket(socket.AF_INET, socket.SOCK_STREAM) http.settimeout(3) http.connect((parse.hostname, parse.port if parse.port else 80)) else: http = ssl.wrap_socket(socket.socket()) http.settimeout(3) http.connect((parse.hostname, 443)) parse = urlparse(url) message = '{0} {1} {2}\r\nHost: {3}\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36\r\nAccept: */*\r\nConnection: close\r\n\r\n' path = parse.path + ( ("?" + parse.query) if parse.query else "" if parse.path else "/") message = message.format(mode, path, Agreement, parse.hostname) http.send(message) response = http.recv(2048) response = response.lower() http.close() headers = AttribDict() headers.code = response[response.find(" ") + 1:response.find("\r\n")] headers.headers = map( lambda x: x.split(":")[0].strip(), response[response.find("\r\n") + 2:response.find("\r\n\r\n")].split("\r\n")) return headers except: return False
def getResponse(req, redirect=False): if type(req) == str: req = req if urlparse(req).scheme != "" else "http://" + req try: resp = urllib2.urlopen(req, timeout=4) while resp.code == 302 and redirect: print resp.headers resp = urllib2.urlopen(resp.headers["Location"], timeout=4) return resp except urllib2.HTTPError, e: return e except: e = AttribDict() e.code = 404 return e def getRequest(url, headers={}, data={}): url = url if urlparse(url).scheme != "" else "http://" + url try: if data: return urllib2.Request(url, headers=headers, data=data) else: return urllib2.Request(url, headers=headers) except: return url def getResponseHeaders(url, Agreement="HTTP/1.0", mode="GET"):