def _update_opener(self, drop_tls_level=False): """ Builds and installs a new opener to be used by all future calls to :func:`urllib2.urlopen`. """ handlers = [ urllib_request.HTTPCookieProcessor(self._cj), urllib_request.HTTPBasicAuthHandler() ] if self._http_debug: handlers += [urllib_request.HTTPHandler(debuglevel=1)] else: handlers += [urllib_request.HTTPHandler()] if self._proxy: handlers += [urllib_request.ProxyHandler({'http': self._proxy})] try: import platform node = platform.node().lower() except: node = '' if not self._ssl_verify or node == 'xboxone': try: import ssl ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE if self._http_debug: handlers += [ urllib_request.HTTPSHandler(context=ctx, debuglevel=1) ] else: handlers += [urllib_request.HTTPSHandler(context=ctx)] except: pass else: try: import ssl import certifi ctx = ssl.create_default_context(cafile=certifi.where()) if drop_tls_level: ctx.protocol = ssl.PROTOCOL_TLSv1_1 if self._http_debug: handlers += [ urllib_request.HTTPSHandler(context=ctx, debuglevel=1) ] else: handlers += [urllib_request.HTTPSHandler(context=ctx)] except: pass opener = urllib_request.build_opener(*handlers) urllib_request.install_opener(opener)
def request(url, close=True, redirect=True, error=False, proxy=None, post=None, headers=None, mobile=False, limit=None, referer=None, cookie=None, output='', timeout='30'): handlers = [] if proxy is not None: handlers += [ ProxyHandler({'http': '{0}'.format(proxy)}), urllib_request.HTTPHandler ] opener = urllib_request.build_opener(*handlers) urllib_request.install_opener(opener) if output == 'cookie' or output == 'extended' or close is not True: cookies = cookielib.LWPCookieJar() handlers += [ urllib_request.HTTPHandler(), urllib_request.HTTPSHandler(), urllib_request.HTTPCookieProcessor(cookies) ] opener = urllib_request.build_opener(*handlers) urllib_request.install_opener(opener) try: if sys.version_info < (2, 7, 9): raise Exception() import ssl ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE handlers += [urllib_request.HTTPSHandler(context=ssl_context)] opener = urllib_request.build_opener(*handlers) urllib_request.install_opener(opener) except: pass try: headers.update(headers) except: headers = {} if 'User-Agent' in headers: pass elif not mobile is True: # headers['User-Agent'] = agent() headers['User-Agent'] = cache.get(randomagent, 1) else: headers['User-Agent'] = 'Apple-iPhone/701.341' if 'Referer' in headers: pass elif referer is None: headers['Referer'] = '%s://%s/' % (urlparse(url).scheme, urlparse(url).netloc) else: headers['Referer'] = referer if not 'Accept-Language' in headers: headers['Accept-Language'] = 'en-US' if 'Cookie' in headers: pass elif cookie is not None: headers['Cookie'] = cookie if redirect is False: class NoRedirection(urllib_error.HTTPError): def http_response(self, request, response): return response opener = urllib_request.build_opener(NoRedirection) urllib_request.install_opener(opener) try: del headers['Referer'] except: pass req = urllib_request.Request(url, data=post, headers=headers) try: response = urllib_request.urlopen(req, timeout=int(timeout)) except urllib_error.HTTPError as response: if response.code == 503: if 'cf-browser-verification' in response.read(5242880): netloc = '%s://%s' % (urlparse(url).scheme, urlparse(url).netloc) cf = cache.get(cfcookie, 168, netloc, headers['User-Agent'], timeout) headers['Cookie'] = cf request = urllib_request.Request(url, data=post, headers=headers) response = urllib_request.urlopen(request, timeout=int(timeout)) elif error is False: return elif error is False: return if output == 'cookie': try: result = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies]) except: pass try: result = cf except: pass elif output == 'response': if limit == '0': result = (str(response.code), response.read(224 * 1024)) elif limit is not None: result = (str(response.code), response.read(int(limit) * 1024)) else: result = (str(response.code), response.read(5242880)) elif output == 'chunk': try: content = int(response.headers['Content-Length']) except: content = (2049 * 1024) if content < (2048 * 1024): return result = response.read(16 * 1024) elif output == 'extended': try: cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies]) except: pass try: cookie = cf except: pass content = response.headers result = response.read(5242880) return result, headers, content, cookie elif output == 'geturl': result = response.geturl() elif output == 'headers': content = response.headers return content else: if limit == '0': result = response.read(224 * 1024) elif limit is not None: result = response.read(int(limit) * 1024) else: result = response.read(5242880) if close is True: response.close() return result
def cfcookie(netloc, ua, timeout): try: headers = {'User-Agent': ua} req = urllib_request.Request(netloc, headers=headers) try: urllib_request.urlopen(req, timeout=int(timeout)) except urllib_request.HTTPError as response: result = response.read(5242880) jschl = re.findall('name="jschl_vc" value="(.+?)"/>', result)[0] init = re.findall('setTimeout\(function\(\){\s*.*?.*:(.*?)};', result)[-1] builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result)[0] decryptVal = parseJSString(init) lines = builder.split(';') for line in lines: if len(line) > 0 and '=' in line: sections = line.split('=') line_val = parseJSString(sections[1]) decryptVal = int( eval(str(decryptVal) + sections[0][-1] + str(line_val))) answer = decryptVal + len(urlparse.urlparse(netloc).netloc) query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % ( netloc, jschl, answer) if 'type="hidden" name="pass"' in result: passval = re.findall('name="pass" value="(.*?)"', result)[0] query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % ( netloc, quote_plus(passval), jschl, answer) time.sleep(5) cookies = cookielib.LWPCookieJar() handlers = [ urllib_request.HTTPHandler(), urllib_request.HTTPSHandler(), urllib_request.HTTPCookieProcessor(cookies) ] opener = urllib_request.build_opener(*handlers) urllib_request.install_opener(opener) try: request = urllib_request.Request(query, headers=headers) urllib_request.urlopen(request, timeout=int(timeout)) except: pass cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies]) return cookie except: pass
def __test_stream(self, stream_url): """ Returns True if the stream_url gets a non-failure http status (i.e. <400) back from the server otherwise return False Intended to catch stream urls returned by resolvers that would fail to playback """ # parse_qsl doesn't work because it splits elements by ';' which can be in a non-quoted UA try: headers = dict([ item.split('=') for item in (stream_url.split('|')[1]).split('&') ]) except: headers = {} for header in headers: headers[header] = urllib_parse.unquote_plus(headers[header]) common.logger.log_debug('Setting Headers on UrlOpen: %s' % headers) try: import ssl ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE opener = urllib_request.build_opener( urllib_request.HTTPSHandler(context=ssl_context)) urllib_request.install_opener(opener) except: pass try: msg = '' request = urllib_request.Request(stream_url.split('|')[0], headers=headers) # only do a HEAD request. gujal request.get_method = lambda: 'HEAD' # set urlopen timeout to 15 seconds http_code = urllib_request.urlopen(request, timeout=15).getcode() except urllib_error.HTTPError as e: if isinstance(e, urllib_error.HTTPError): http_code = e.code if http_code == 405: http_code = 200 else: http_code = 600 except urllib_error.URLError as e: http_code = 500 if hasattr(e, 'reason'): # treat an unhandled url type as success if 'unknown url type' in str(e.reason).lower(): return True else: msg = e.reason if not msg: msg = str(e) except Exception as e: http_code = 601 msg = str(e) if msg == "''": http_code = 504 # added this log line for now so that we can catch any logs on streams that are rejected due to test_stream failures # we can remove it once we are sure this works reliably if int(http_code) >= 400 and int(http_code) != 504: common.logger.log_warning( 'Stream UrlOpen Failed: Url: %s HTTP Code: %s Msg: %s' % (stream_url, http_code, msg)) return int(http_code) < 400 or int(http_code) == 504
base_hdrs = {'User-Agent': USER_AGENT, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 'Accept-Encoding': 'gzip', 'Accept-Language': 'en-US,en;q=0.8', 'Connection': 'keep-alive'} openloadhdr = base_hdrs progress = xbmcgui.DialogProgress() dialog = xbmcgui.Dialog() urlopen = urllib_request.urlopen cj = http_cookiejar.LWPCookieJar(TRANSLATEPATH(cookiePath)) Request = urllib_request.Request handlers = [urllib_request.HTTPBasicAuthHandler(), urllib_request.HTTPHandler(), urllib_request.HTTPSHandler()] ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE handlers.append(urllib_request.HTTPSHandler(context=ssl_context)) def kodilog(logvar, level=LOGINFO): xbmc.log("@@@@Cumination: " + str(logvar), level) @url_dispatcher.register() def clear_cache(): """ Clear the cache database. """
'Accept-Language': 'en-US,en;q=0.8', 'Connection': 'keep-alive' } openloadhdr = base_hdrs progress = xbmcgui.DialogProgress() dialog = xbmcgui.Dialog() urlopen = urllib_request.urlopen cj = http_cookiejar.LWPCookieJar(TRANSLATEPATH(cookiePath)) Request = urllib_request.Request handlers = [ urllib_request.HTTPBasicAuthHandler(), urllib_request.HTTPHandler(), urllib_request.HTTPSHandler() ] if (2, 7, 8) < sys.version_info: try: ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE handlers += [urllib_request.HTTPSHandler(context=ssl_context)] except: pass def kodilog(logvar, level=LOGINFO): xbmc.log("@@@@Cumination: " + str(logvar), level)
'Accept-Language': 'en-US,en;q=0.8', 'Connection': 'keep-alive' } openloadhdr = base_hdrs progress = xbmcgui.DialogProgress() dialog = xbmcgui.Dialog() urlopen = urllib_request.urlopen cj = http_cookiejar.LWPCookieJar(TRANSLATEPATH(cookiePath)) Request = urllib_request.Request handlers = [ urllib_request.HTTPBasicAuthHandler(), urllib_request.HTTPHandler(), urllib_request.HTTPSHandler() ] ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE handlers.append(urllib_request.HTTPSHandler(context=ssl_context)) def kodilog(logvar, level=LOGINFO): xbmc.log("@@@@Cumination: " + str(logvar), level) @url_dispatcher.register() def clear_cache(): """ Clear the cache database.