def connection(self, url, payload=""):
time.sleep(conf.delay_time)
ret = AttribDict()
ret.code = 0
ret.text = ''
ret.success = False
if conf.raw:
raw_data = self.raw_request
start_time = time.time()
status_code, head, content, redirect, log = self.send_raw_request(
url, raw_data, payload)
end_time = time.time()
self.elapsed_time = end_time - start_time
ret.code = status_code
con = content
con = format_unicode(con)
ret.text = get_text_value(con)
ret.success = True
else:
if self.method == 'get':
payload = url_encode(payload)
pass
try:
url, params, data = self.generate_params(url, payload)
if data:
data = dict_to_q_str(data)
if self.method == 'get':
start_time = time.time()
status_code, head, content, redirect, log = fly_req.http(
url,
timeout=self.timeout,
header=self.headers,
proxy=self.proxies)
end_time = time.time()
self.elapsed_time = end_time - start_time
ret.code = status_code
con = content
con = format_unicode(con)
ret.text = get_text_value(con)
ret.success = True
if self.method == 'post':
start_time = time.time()
status_code, head, content, redirect, log = fly_req.http(
url + '?' + params,
data=data,
timeout=self.timeout,
header=self.headers,
proxy=self.proxies)
end_time = time.time()
self.elapsed_time = end_time - start_time
ret.code = status_code
con = content
con = format_unicode(con)
ret.text = get_text_value(con)
ret.success = True
except Exception as e:
logger.warning(e.message)
sys.exit(0)
return ret
def getResponseHeaders(url, Agreement="HTTP/1.0", mode="GET"):
#返回response中headers的排序list和code (均小写)
try:
parse = urlparse(url)
url = url if parse.scheme != "" else "http://" + url
if parse.scheme == "http":
http = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
http.settimeout(3)
http.connect((parse.hostname, parse.port if parse.port else 80))
else:
http = ssl.wrap_socket(socket.socket())
http.settimeout(3)
http.connect((parse.hostname, 443))
parse = urlparse(url)
message = '{0} {1} {2}\r\nHost: {3}\r\nUser-Agent: Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36\r\nAccept: */*\r\nConnection: close\r\n\r\n'
path = parse.path + (
("?" + parse.query) if parse.query else "" if parse.path else "/")
message = message.format(mode, path, Agreement, parse.hostname)
http.send(message)
response = http.recv(2048)
response = response.lower()
http.close()
headers = AttribDict()
headers.code = response[response.find(" ") + 1:response.find("\r\n")]
headers.headers = map(
lambda x: x.split(":")[0].strip(),
response[response.find("\r\n") +
2:response.find("\r\n\r\n")].split("\r\n"))
return headers
except:
return False
def getResponse(req, redirect=False):
if type(req) == str:
req = req if urlparse(req).scheme != "" else "http://" + req
try:
resp = urllib2.urlopen(req, timeout=4)
while resp.code == 302 and redirect:
print resp.headers
resp = urllib2.urlopen(resp.headers["Location"], timeout=4)
return resp
except urllib2.HTTPError, e:
return e
except:
e = AttribDict()
e.code = 404
return e
def getRequest(url, headers={}, data={}):
url = url if urlparse(url).scheme != "" else "http://" + url
try:
if data:
return urllib2.Request(url, headers=headers, data=data)
else:
return urllib2.Request(url, headers=headers)
except:
return url
def getResponseHeaders(url, Agreement="HTTP/1.0", mode="GET"):
