def __init__(self,url,callback,method='GET',headers=None,need_proxy=False,fail_time=0,timeout=TIMEOUT):
# super(WeixinRequest,self).__init__(method,url,headers) #继承父类的顺序不能乱
Request.__init__(self,method,url,headers)
self.callback=callback
self.need_proxy=need_proxy
self.fail_time=fail_time
self.timeout=timeout
def __init__(self, url, callback, method='GET', headers=None, timeout=3, proxy=None, params=None, data=None):
Request.__init__(url, method, headers)
self.callback = callback
self.timeout = timeout
self.proxy = proxy
self.params = params
self.data = data
def __init__(self, url, callback, method='GET', header=None, need_proxy=False, fail_time=0,
timeout=TIMEOUT):
Request.__init__(self, method, url, header)
self.callback = callback
self.need_proxy = need_proxy
self.fail_time = fail_time
self.timeout = timeout
def __init__(self, url=None, callback, method='GET',
headers=None, need_proxy=False, fail_times=0, timeout=TIMEOUT):
Request.__init__(self, method, url, headers)
self.callback = callback
self.need_proxy = need_proxy
self.fail_times = fail_times
self.timeout = timeout
def __init__(self, url, callback, method='GET', headers=None, need_proxy=False, fail_time=0, timeout=5):
if headers is None:
headers = self.DEFAULT_HEADERS
Request.__init__(self, method, url, headers)
self.callback = callback
self.need_proxy = need_proxy
self.fail_time = fail_time
self.timeout = timeout
def __init__(self: HTTPEntity, cert: Dict = None):
"""HTTPEntity base class constructor
Args:
cert (Dict): certificate for authentication.
"""
Request.__init__(self)
return
def __init__(self,
method='GET', url=None, headers=None, files=None, data=None,
params=None, auth=None, cookies=None, hooks=None, json=None,
call_back=None, need_proxy=False, failtime=0, timeout=10):
Request.__init__(self, method, url, headers)
self.call_back = call_back
self.need_proxy = need_proxy
self.failtime = failtime
self.timeout = timeout
def __init__(self: HTTPEntity, cert: Dict=None):
"""HTTPEntity base class constructor
Args:
cert (Dict): certificate for authentication.
"""
warnings.filterwarnings(action="ignore", message="unclosed", category=ResourceWarning)
Request.__init__(self)
return
def __init__(self,
method='GET',
callback=None,
need_proxy=False,
fail_time=0,
*args,
**kwargs):
Request.__init__(self, method, *args, **kwargs)
self.callback = callback
self.need_proxy = need_proxy
self.fail_time = fail_time
def __init__(self,
url,
callback,
method='GET',
headers=None,
need_proxy=False,
fail_time=0,
timeout=TIMEOUT):
Request.__init__(self, method, url, headers)
self.callback = callback #回调函数
self.need_proxy = need_proxy #是否需要代理爬取
self.fail_time = fail_time #失败次数
self.timeout = timeout #超时时间
def __init__(self,
url,
method='GET',
headers=None,
callback=None,
need_proxy=None,
timeout=TIMEOUT,
failed_time=0):
Request.__init__(self, method, url, headers)
self.need_proxy = need_proxy
self.callback = callback
self.failed_time = failed_time
self.timeout = timeout
def __init__(self,
url,
callback,
method='GET',
headers=None,
need_proxy=False,
fail_time=0,
timeout=TIMEOUT):
#注意此处三个参数顺序不能写错,或者写成url=url的形式
Request.__init__(self, method, url, headers)
self.callback = callback
self.need_proxy = need_proxy
self.fail_time = fail_time
self.timeout = timeout
def __init__(self,
url,
callback,
method='GET',
headers=None,
timeout=3,
proxy=None,
params=None,
data=None):
Request.__init__(url, method, headers)
self.callback = callback
self.timeout = timeout
self.proxy = proxy
self.params = params
self.data = data
def __init__(self,
url,
callback,
method='GET',
headers=None,
need_proxy=False,
fail_time=0,
timeout=TIMEOUT):
# 继承调用Request中的方法
Request.__init__(self, method, url, headers)
# 添加额外的参数 +--
self.callback = callback
self.need_proxy = need_proxy
self.fail_time = fail_time
self.timeout = timeout
def __init__(self, method=None, url=None, proxy_name=None, headers=None, files=None,
data=None, params=None, auth=None, cookies=None, hooks=None,
crawler_name=None, timeout=None, json=None):
'''This this the `Reqeust` class in ysacrapy.
:param method: string, http method such as "GET", "POST", etc.
:param url: string, http request url to use.
:param proxy_name: string, get http proxy from proxy set using this proxy_name.
:param headers: dict, http request headers.
:param data: string, use data if `method` is "POST".
:param cookies: dict, http request cookies.
:param crawler_name: string, crawler name to use with this `Request` object.
Use `requests.Request` as base class to keep interface simple.
Just use this class like `requests.Request`.
'''
RequestLib.__init__(
self,
method=method,
url=url,
headers=headers,
files=files,
data=data,
params=params,
auth=auth,
cookies=cookies,
hooks=hooks,
json=json
)
self.data = data if data else ''
self.crawler_name = crawler_name
self.timeout = timeout
self.proxy_name = proxy_name
self.method = method if method else 'GET'
self.timeout = timeout if timeout else 15
def __init__(self,
url,
callback,
method="GET",
headers=None,
need_proxy=False,
fail_time=0,
timeout=TIMEOUT):
"""
:param url: 请求的URL
:param callback: 回调函数
:param method: 默认 GET 请求
:param headers: 默认无请求头
:param need_proxy: 默认不使用代理
:param fail_time: 失败重试次数
:param timeout: 超时时间
"""
Request.__init__(self, method, url, headers)
self.callback = callback
self.need_proxy = need_proxy
self.fail_time = fail_time
self.timeout = timeout
def __init__(self,
url,
callback,
method='GET',
headers=None,
need_proxy=NEED_PROXY,
timeout=TIMEOUT,
fail_time=0):
"""
:param url: url
:param callback: 回调函数
:param method: 请求方法
:param headers: 请求头
:param need_proxy: 是否需要代理
:param timeout: 超时时间
:param fail_time: 请求失败次数
"""
Request.__init__(self, method, url, headers)
self.callback = callback
self.need_proxy = need_proxy
self.timeout = timeout
self.fail_time = fail_time
def __init__(self, url, callback, method='GET', headers=None, need_proxy=False, fail_time=0, timeout=TIMEOUT):
Request.__init__(self, method, url, headers)
self.callback = callback # 加入额外参数callback回调函数：知道请求的响应用什么方法来处理
self.need_proxy = need_proxy # 是否需要代理爬取
self.fail_time = fail_time # 失败次数：可以知道请求失败了多少次，判断失败次数是否到了阈值，该不该丢弃这个请求
self.timeout = timeout # 超时时间
