def __init__(self,WEBCONFIG=WEBCONFIG,debuglevel=0):
self.__RedirectHandler=webtool.RedirectHandler()
self.__encoding_support = gzipsupport.ContentEncodingProcessor()
"""
启用代理模块
"""
self.__enable_proxy=WEBCONFIG.enable_proxy
self.__proxy_handler= urllib2.ProxyHandler({WEBCONFIG.proxy_name:WEBCONFIG.proxy_address})
self.__null_proxy_handler= urllib2.ProxyHandler({})
urllib2.socket.setdefaulttimeout(WEBCONFIG.time_out) #设置超时时间
self.__headers = {
'User-Agent' : WEBCONFIG.useragent,
'Referer': WEBCONFIG.Referer
}
self.__cookie=cookielib.CookieJar()
self.__cJar=cookielib.LWPCookieJar()
self.__httpcookieprocessor=urllib2.HTTPCookieProcessor(self.__cookie)
self.__httpHandler= urllib2.HTTPHandler(debuglevel=debuglevel)
self.__httpsHandler=urllib2.HTTPSHandler(debuglevel=debuglevel)
self.__opener=''
if self.__enable_proxy:
self.__opener=urllib2.build_opener(self.__encoding_support,self.__httpcookieprocessor,self.__proxy_handler,self.__httpHandler,self.__httpsHandler,self.__RedirectHandler)
else:
self.__opener=urllib2.build_opener(self.__encoding_support,self.__httpcookieprocessor,self.__null_proxy_handler,self.__httpHandler,self.__httpsHandler,self.__RedirectHandler)
urllib2.install_opener(self.__opener)
def __init__(self, WEBCONFIG=WEBCONFIG, debuglevel=0):
# 声明一些问题在301/302
self.__RedirectHandler = webtool.RedirectHandler()
self.__encoding_support = gzipsupport.ContentEncodingProcessor()
"""
启用代理模块
"""
self.__enable_proxy = WEBCONFIG.enable_proxy #False
# proxy_name:http, proxy_address:http://abc.com:80
# http:http://abc.com:8
self.__proxy_handler= urllib2.ProxyHandler({WEBCONFIG.proxy_name:WEBCONFIG.proxy_address})
self.__null_proxy_handler= urllib2.ProxyHandler({})
#设置超时时间10s
urllib2.socket.setdefaulttimeout(WEBCONFIG.time_out)
self.__headers = {
'User-Agent' : WEBCONFIG.useragent,
'Referer': WEBCONFIG.Referer
}
self.__cookie = cookielib.CookieJar()
self.__cJar = cookielib.LWPCookieJar()
self.__httpcookieprocessor = urllib2.HTTPCookieProcessor(self.__cookie)
self.__httpHandler = urllib2.HTTPHandler(debuglevel=debuglevel)
self.__httpsHandler = urllib2.HTTPSHandler(debuglevel=debuglevel)
self.__opener=''
if self.__enable_proxy:
self.__opener = urllib2.build_opener(self.__encoding_support,self.__httpcookieprocessor,self.__proxy_handler,self.__httpHandler,self.__httpsHandler,self.__RedirectHandler)
else:
self.__opener = urllib2.build_opener(self.__encoding_support,self.__httpcookieprocessor,self.__null_proxy_handler,self.__httpHandler,self.__httpsHandler,self.__RedirectHandler)
# urllib2.install_opener() 会设置 urllib2 的全局 opener
# 这样后面的使用会很方便，但不能做更细致的控制，比如想在程序中使用两个不同的 Proxy 设置等。比较好的做法是不使用 install_opener 去更改全局的设置，而只是直接调用 opener 的 open 方法代替全局的 urlopen 方法
urllib2.install_opener(self.__opener)
import urllib
import urllib2
import cookielib
import re
import SQLtool
import webconfig
import sys
import webtool
import gzipsupport
from threading import stack_size
import time
reload(sys)
stack_size(32768 * 16)
WEBCONFIG = webconfig.WebConfig
RedirectHandler = webtool.RedirectHandler()
encoding_support = gzipsupport.ContentEncodingProcessor()
# values ={}
# values['name']='123'
# values = {'name' : 'Michael Foord', 'location' : 'Northampton', 'language' : 'Python' }
def gethtml(URL, way, params={}, times=1, timeout=10):
"""
启用代理模块
"""
enable_proxy = WEBCONFIG.enable_proxy
proxy_handler = urllib2.ProxyHandler(
{WEBCONFIG.proxy_name: WEBCONFIG.proxy_address})
null_proxy_handler = urllib2.ProxyHandler({})
urllib2.socket.setdefaulttimeout(timeout) #设置超时时间
url = URL