def make_request(method, url, headers, body):
domain = get_domain_from_url(url)
if headers:
# delete some needless header
for key in headers.keys():
if key in [
'Accept-Encoding', 'Content-Type', 'Accept-Language',
'Accept', 'Connection'
]:
del headers[key]
else:
headers = getheader_dict(domain)
# proxy(127.0.0.1:8080)
# opener=urllib2.build_opener(proxy_support)
# opener = urllib2.build_opener()
# opener.addheaders=headers
# urllib2.install_opener(opener)
if method == 'GET':
req = urllib2.Request(url, headers=headers)
try:
resp = urllib2.urlopen(req)
# save redirect
if resp.url != url:
REDIRECT.append(url)
return resp
except URLError, e:
REQUEST_ERROR.append(('make_request()', url, e.reason))
except CertificateError:
REQUEST_ERROR.append(
('make_request()', url, 'ssl.CertificateError'))
try:
resp = urllib2.urlopen(req)
except urllib2.URLError, e:
REQUEST_ERROR.append(('gen_traffic()', url, e.reason))
except CertificateError:
REQUEST_ERROR.append(
('gen_traffic()', url, 'ssl.CertificateError'))
except ValueError, e:
print e
except BadStatusLine, e:
print e
except SocketError, e:
print e
else:
if resp.url != url:
REDIRECT.append(url)
try:
data = resp.read()
except Exception, e:
print e
else:
resp_headers = resp.headers.headers
resp_headers_dict = list2dict(resp_headers)
response = HttpResponse(code=str(resp.code),
reason=resp.msg,
headers=resp_headers_dict,
data=data)
return (request, response)
def run(self):
import gevent