def _update_opener(self, drop_tls_level=False):
"""
Builds and installs a new opener to be used by all future calls to
:func:`urllib2.urlopen`.
"""
handlers = [
urllib_request.HTTPCookieProcessor(self._cj),
urllib_request.HTTPBasicAuthHandler()
]
if self._http_debug:
handlers += [urllib_request.HTTPHandler(debuglevel=1)]
else:
handlers += [urllib_request.HTTPHandler()]
if self._proxy:
handlers += [urllib_request.ProxyHandler({'http': self._proxy})]
try:
import platform
node = platform.node().lower()
except:
node = ''
if not self._ssl_verify or node == 'xboxone':
try:
import ssl
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
if self._http_debug:
handlers += [
urllib_request.HTTPSHandler(context=ctx, debuglevel=1)
]
else:
handlers += [urllib_request.HTTPSHandler(context=ctx)]
except:
pass
else:
try:
import ssl
import certifi
ctx = ssl.create_default_context(cafile=certifi.where())
if drop_tls_level:
ctx.protocol = ssl.PROTOCOL_TLSv1_1
if self._http_debug:
handlers += [
urllib_request.HTTPSHandler(context=ctx, debuglevel=1)
]
else:
handlers += [urllib_request.HTTPSHandler(context=ctx)]
except:
pass
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
def request(url,
close=True,
redirect=True,
error=False,
proxy=None,
post=None,
headers=None,
mobile=False,
limit=None,
referer=None,
cookie=None,
output='',
timeout='30'):
handlers = []
if proxy is not None:
handlers += [
ProxyHandler({'http': '{0}'.format(proxy)}),
urllib_request.HTTPHandler
]
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
if output == 'cookie' or output == 'extended' or close is not True:
cookies = cookielib.LWPCookieJar()
handlers += [
urllib_request.HTTPHandler(),
urllib_request.HTTPSHandler(),
urllib_request.HTTPCookieProcessor(cookies)
]
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
try:
if sys.version_info < (2, 7, 9):
raise Exception()
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib_request.HTTPSHandler(context=ssl_context)]
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
except:
pass
try:
headers.update(headers)
except:
headers = {}
if 'User-Agent' in headers:
pass
elif not mobile is True:
# headers['User-Agent'] = agent()
headers['User-Agent'] = cache.get(randomagent, 1)
else:
headers['User-Agent'] = 'Apple-iPhone/701.341'
if 'Referer' in headers:
pass
elif referer is None:
headers['Referer'] = '%s://%s/' % (urlparse(url).scheme,
urlparse(url).netloc)
else:
headers['Referer'] = referer
if not 'Accept-Language' in headers:
headers['Accept-Language'] = 'en-US'
if 'Cookie' in headers:
pass
elif cookie is not None:
headers['Cookie'] = cookie
if redirect is False:
class NoRedirection(urllib_error.HTTPError):
def http_response(self, request, response):
return response
opener = urllib_request.build_opener(NoRedirection)
urllib_request.install_opener(opener)
try:
del headers['Referer']
except:
pass
req = urllib_request.Request(url, data=post, headers=headers)
try:
response = urllib_request.urlopen(req, timeout=int(timeout))
except urllib_error.HTTPError as response:
if response.code == 503:
if 'cf-browser-verification' in response.read(5242880):
netloc = '%s://%s' % (urlparse(url).scheme,
urlparse(url).netloc)
cf = cache.get(cfcookie, 168, netloc, headers['User-Agent'],
timeout)
headers['Cookie'] = cf
request = urllib_request.Request(url,
data=post,
headers=headers)
response = urllib_request.urlopen(request,
timeout=int(timeout))
elif error is False:
return
elif error is False:
return
if output == 'cookie':
try:
result = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
result = cf
except:
pass
elif output == 'response':
if limit == '0':
result = (str(response.code), response.read(224 * 1024))
elif limit is not None:
result = (str(response.code), response.read(int(limit) * 1024))
else:
result = (str(response.code), response.read(5242880))
elif output == 'chunk':
try:
content = int(response.headers['Content-Length'])
except:
content = (2049 * 1024)
if content < (2048 * 1024):
return
result = response.read(16 * 1024)
elif output == 'extended':
try:
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
except:
pass
try:
cookie = cf
except:
pass
content = response.headers
result = response.read(5242880)
return result, headers, content, cookie
elif output == 'geturl':
result = response.geturl()
elif output == 'headers':
content = response.headers
return content
else:
if limit == '0':
result = response.read(224 * 1024)
elif limit is not None:
result = response.read(int(limit) * 1024)
else:
result = response.read(5242880)
if close is True:
response.close()
return result
def cfcookie(netloc, ua, timeout):
try:
headers = {'User-Agent': ua}
req = urllib_request.Request(netloc, headers=headers)
try:
urllib_request.urlopen(req, timeout=int(timeout))
except urllib_request.HTTPError as response:
result = response.read(5242880)
jschl = re.findall('name="jschl_vc" value="(.+?)"/>', result)[0]
init = re.findall('setTimeout\(function\(\){\s*.*?.*:(.*?)};',
result)[-1]
builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result)[0]
decryptVal = parseJSString(init)
lines = builder.split(';')
for line in lines:
if len(line) > 0 and '=' in line:
sections = line.split('=')
line_val = parseJSString(sections[1])
decryptVal = int(
eval(str(decryptVal) + sections[0][-1] + str(line_val)))
answer = decryptVal + len(urlparse.urlparse(netloc).netloc)
query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % (
netloc, jschl, answer)
if 'type="hidden" name="pass"' in result:
passval = re.findall('name="pass" value="(.*?)"', result)[0]
query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % (
netloc, quote_plus(passval), jschl, answer)
time.sleep(5)
cookies = cookielib.LWPCookieJar()
handlers = [
urllib_request.HTTPHandler(),
urllib_request.HTTPSHandler(),
urllib_request.HTTPCookieProcessor(cookies)
]
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
try:
request = urllib_request.Request(query, headers=headers)
urllib_request.urlopen(request, timeout=int(timeout))
except:
pass
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
return cookie
except:
pass
def __test_stream(self, stream_url):
"""
Returns True if the stream_url gets a non-failure http status (i.e. <400) back from the server
otherwise return False
Intended to catch stream urls returned by resolvers that would fail to playback
"""
# parse_qsl doesn't work because it splits elements by ';' which can be in a non-quoted UA
try:
headers = dict([
item.split('=')
for item in (stream_url.split('|')[1]).split('&')
])
except:
headers = {}
for header in headers:
headers[header] = urllib_parse.unquote_plus(headers[header])
common.logger.log_debug('Setting Headers on UrlOpen: %s' % headers)
try:
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
opener = urllib_request.build_opener(
urllib_request.HTTPSHandler(context=ssl_context))
urllib_request.install_opener(opener)
except:
pass
try:
msg = ''
request = urllib_request.Request(stream_url.split('|')[0],
headers=headers)
# only do a HEAD request. gujal
request.get_method = lambda: 'HEAD'
# set urlopen timeout to 15 seconds
http_code = urllib_request.urlopen(request, timeout=15).getcode()
except urllib_error.HTTPError as e:
if isinstance(e, urllib_error.HTTPError):
http_code = e.code
if http_code == 405:
http_code = 200
else:
http_code = 600
except urllib_error.URLError as e:
http_code = 500
if hasattr(e, 'reason'):
# treat an unhandled url type as success
if 'unknown url type' in str(e.reason).lower():
return True
else:
msg = e.reason
if not msg:
msg = str(e)
except Exception as e:
http_code = 601
msg = str(e)
if msg == "''":
http_code = 504
# added this log line for now so that we can catch any logs on streams that are rejected due to test_stream failures
# we can remove it once we are sure this works reliably
if int(http_code) >= 400 and int(http_code) != 504:
common.logger.log_warning(
'Stream UrlOpen Failed: Url: %s HTTP Code: %s Msg: %s' %
(stream_url, http_code, msg))
return int(http_code) < 400 or int(http_code) == 504
base_hdrs = {'User-Agent': USER_AGENT,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
'Accept-Encoding': 'gzip',
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'}
openloadhdr = base_hdrs
progress = xbmcgui.DialogProgress()
dialog = xbmcgui.Dialog()
urlopen = urllib_request.urlopen
cj = http_cookiejar.LWPCookieJar(TRANSLATEPATH(cookiePath))
Request = urllib_request.Request
handlers = [urllib_request.HTTPBasicAuthHandler(), urllib_request.HTTPHandler(), urllib_request.HTTPSHandler()]
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers.append(urllib_request.HTTPSHandler(context=ssl_context))
def kodilog(logvar, level=LOGINFO):
xbmc.log("@@@@Cumination: " + str(logvar), level)
@url_dispatcher.register()
def clear_cache():
"""
Clear the cache database.
"""
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'
}
openloadhdr = base_hdrs
progress = xbmcgui.DialogProgress()
dialog = xbmcgui.Dialog()
urlopen = urllib_request.urlopen
cj = http_cookiejar.LWPCookieJar(TRANSLATEPATH(cookiePath))
Request = urllib_request.Request
handlers = [
urllib_request.HTTPBasicAuthHandler(),
urllib_request.HTTPHandler(),
urllib_request.HTTPSHandler()
]
if (2, 7, 8) < sys.version_info:
try:
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers += [urllib_request.HTTPSHandler(context=ssl_context)]
except:
pass
def kodilog(logvar, level=LOGINFO):
xbmc.log("@@@@Cumination: " + str(logvar), level)
'Accept-Language': 'en-US,en;q=0.8',
'Connection': 'keep-alive'
}
openloadhdr = base_hdrs
progress = xbmcgui.DialogProgress()
dialog = xbmcgui.Dialog()
urlopen = urllib_request.urlopen
cj = http_cookiejar.LWPCookieJar(TRANSLATEPATH(cookiePath))
Request = urllib_request.Request
handlers = [
urllib_request.HTTPBasicAuthHandler(),
urllib_request.HTTPHandler(),
urllib_request.HTTPSHandler()
]
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
handlers.append(urllib_request.HTTPSHandler(context=ssl_context))
def kodilog(logvar, level=LOGINFO):
xbmc.log("@@@@Cumination: " + str(logvar), level)
@url_dispatcher.register()
def clear_cache():
"""
Clear the cache database.
