def _http_get(self, url, data=None, headers=None, cache_limit=8):
html = self._cached_http_get(url, self.base_url, self.timeout, data=data, cache_limit=cache_limit)
cookie = scraper_utils.get_sucuri_cookie(html)
if cookie:
log_utils.log('Setting Pubfilm cookie: %s' % (cookie), log_utils.LOGDEBUG)
html = self._cached_http_get(url, self.base_url, self.timeout, cookies=cookie, data=data, headers=headers, cache_limit=0)
return html
def _http_get(self, url, data=None, headers=None, cache_limit=8):
html = self._cached_http_get(url, self.base_url, self.timeout, data=data, headers=headers, cache_limit=cache_limit)
cookie = scraper_utils.get_sucuri_cookie(html)
if cookie:
log_utils.log('Setting Zumvo cookie: %s' % (cookie), log_utils.LOGDEBUG)
html = self._cached_http_get(url, self.base_url, self.timeout, cookies=cookie, data=data, headers=headers, cache_limit=0)
return html
def _http_get(self, url, params=None, data=None, multipart_data=None, headers=None, cookies=None, allow_redirect=True, method=None, require_debrid=False, read_error=False, cache_limit=8):
html = self._cached_http_get(url, self.base_url, self.timeout, params=params, data=data, multipart_data=multipart_data,
headers=headers, cookies=cookies, allow_redirect=allow_redirect, method=method, require_debrid=require_debrid,
read_error=read_error, cache_limit=cache_limit)
sucuri_cookie = scraper_utils.get_sucuri_cookie(html)
if sucuri_cookie:
logger.log('Setting sucuri cookie: %s' % (sucuri_cookie), log_utils.LOGDEBUG)
if cookies is not None:
cookies = cookies.update(sucuri_cookie)
else:
cookies = sucuri_cookie
html = self._cached_http_get(url, self.base_url, self.timeout, params=params, data=data, multipart_data=multipart_data,
headers=headers, cookies=cookies, allow_redirect=allow_redirect, method=method, require_debrid=require_debrid,
read_error=read_error, cache_limit=0)
return html
def _http_get(self,
url,
params=None,
data=None,
multipart_data=None,
headers=None,
cookies=None,
allow_redirect=True,
method=None,
require_debrid=False,
read_error=False,
cache_limit=8):
html = self._cached_http_get(url,
self.base_url,
self.timeout,
params=params,
data=data,
multipart_data=multipart_data,
headers=headers,
cookies=cookies,
allow_redirect=allow_redirect,
method=method,
require_debrid=require_debrid,
read_error=read_error,
cache_limit=cache_limit)
sucuri_cookie = scraper_utils.get_sucuri_cookie(html)
if sucuri_cookie:
logger.log('Setting sucuri cookie: %s' % (sucuri_cookie),
log_utils.LOGDEBUG)
if cookies is not None:
cookies = cookies.update(sucuri_cookie)
else:
cookies = sucuri_cookie
html = self._cached_http_get(url,
self.base_url,
self.timeout,
params=params,
data=data,
multipart_data=multipart_data,
headers=headers,
cookies=cookies,
allow_redirect=allow_redirect,
method=method,
require_debrid=require_debrid,
read_error=read_error,
cache_limit=0)
return html
