def get_json(self, url, cookies, referer, form_data=None):
ajax_response, e = self.get_html(url, cookies, referer, form_data)
import simplejson
json = simplejson.loads(ajax_response)
if helper.debug_dump_html():
helper.log_debug('JSON DUMP: %s' % json)
return json
def get_html(self, url, cookies, referer, form_data=None):
html, html_exception = '', None
try:
self.set_cookies(cookies)
helper.log_debug('Performing a %s operation' %
('POST' if form_data else 'GET'))
if form_data:
html = self.http_POST(url,
form_data,
headers={
'Referer': referer
}).content
else:
html = self.http_GET(url, headers={'Referer': referer}).content
if html != '':
helper.log_debug("Saving cookies")
self.save_cookies(cookies)
helper.log_debug("Operation complete")
except Exception as e:
html_exception = e
if helper.debug_dump_html():
helper.log_debug('HTML DUMP: %s' % html)
return (html, html_exception)
def _parseJSString(self, s):
'''
Credit to lambda - https://offshoregit.com/lambda81/
plugin.video.genesis\resources\lib\libraries\cloudflare.py
'''
try:
offset = 1 if s[0] == '+' else 0
val = int(
eval(
s.replace('!+[]', '1').replace('!![]', '1').replace(
'[]', '0').replace('(', 'str(')[offset:]))
return val
except Exception as e:
helper.log_debug('_parseJSString failed with exception %s' %
str(e))
pass
def __update_row(self, id, args):
sql_update = 'INSERT OR REPLACE INTO last_visited '\
'(id, action, value, icon, fanart, full_title, base_title, '\
'imdb_id, tvdb_id, tmdb_id, media_type) '\
'VALUES (%s)' % (', '.join('?' * 11))
# Be sure to decode the names which may contain funky characters!
full_title = args['full_title'].decode(
'utf8') if args['full_title'] else ''
base_title = args['base_title'].decode(
'utf8') if args['base_title'] else ''
data = (id, args['action'], args['value'], args['icon'],
args['fanart'], full_title, base_title, args['imdb_id'],
args['tvdb_id'], args['tmdb_id'], args['media_type'])
helper.log_debug('SQL INSERT OR REPLACE: %s with params %s' %
(sql_update, str(data)))
self.dbcur.execute(sql_update, data)
self.dbcon.commit()
def _resolve_cloudflare(self,
url,
challenge,
form_data={},
headers={},
compression=True):
"""
Asks _cloudflare for an URL with the answer to overcome the
challenge, and then attempts the resolution.
"""
helper.start("_resolve_cloudflare")
from urlparse import urlparse, urlunparse
parsed_url = urlparse(url)
cloudflare_url = urlunparse(
(parsed_url.scheme, parsed_url.netloc, '', '', '', ''))
query = self._get_cloudflare_answer(cloudflare_url, challenge,
form_data, headers, compression)
# Use the cloudflare jar instead for this attempt; revert back to
# main jar after attempt with call to update_opener()
self._update_opener_with_cloudflare()
try:
helper.log_debug("Attempting to resolve the challenge")
response = Net._fetch(self, query, form_data, headers, compression)
helper.log_debug("Resolved the challenge, updating cookies")
for c in self._cloudflare_jar:
self._cj.set_cookie(c)
self._update_opener()
except urllib2.HTTPError as e:
helper.log_debug(
"Failed to resolve the cloudflare challenge with exception %s"
% str(e))
self._update_opener()
pass
helper.end('_resolve_cloudflare')
def __get_row(self, id):
sql_select = 'SELECT * FROM last_visited WHERE id=?'
helper.log_debug('SQL SELECT: %s with params: %s' % (sql_select, id))
self.dbcur.execute(sql_select, (id, ))
matchedrow = self.dbcur.fetchone()
return dict(matchedrow) if matchedrow else None
def _fetch(self, url, form_data={}, headers={}, compression=True):
'''
A wrapper around the super's _fetch with cloudflare support
'''
helper.log_debug("Fetch attempt url: %s, form_data: %s, headers: %s" %
(url, form_data, headers))
if not self._cloudflare:
return Net._fetch(self, url, form_data, headers, compression)
else:
try:
r = Net._fetch(self, url, form_data, headers, compression)
helper.log_debug('Did not encounter a cloudflare challenge')
return r
except urllib2.HTTPError as e:
if e.code == 503:
helper.log_debug('Encountered a cloudflare challenge')
challenge = e.read()
if challenge == 'The service is unavailable.':
helper.log_debug(
'Challenge says the service is unavailable')
raise
try:
helper.log_debug(
"Received a challenge, so we'll need to get around cloudflare"
)
self._resolve_cloudflare(url, challenge, form_data,
headers, compression)
helper.log_debug(
"Successfully resolved cloudflare challenge, fetching real response"
)
return Net._fetch(self, url, form_data, headers,
compression)
except urllib2.HTTPError as e:
helper.log_debug(
"Failed to set up cloudflare with exception %s" %
str(e))
raise
else:
helper.log_debug('Initial attempt failed with code %d' %
e.code)
raise
def _get_cloudflare_answer(self,
url,
challenge,
form_data={},
headers={},
compression=True):
'''
Use the cloudflare cookie jar to overcome the cloudflare challenge.
Returns an URL with the answer to try.
Credit to lambda - https://offshoregit.com/lambda81/
plugin.video.genesis\resources\lib\libraries\cloudflare.py
'''
helper.start("_get_cloudflare_answer")
if not challenge:
helper.log_debug('Challenge is empty, re')
raise ValueError('Challenge is empty')
import re
try:
jschl = re.compile('name="jschl_vc" value="(.+?)"/>').findall(
challenge)[0]
init_str = re.compile('setTimeout\(function\(\){\s*.*?.*:(.*?)};'
).findall(challenge)[0]
builder = re.compile(r"challenge-form\'\);\s*(.*)a.v").findall(
challenge)[0]
decrypt_val = self._parseJSString(init_str)
lines = builder.split(';')
except Exception as e:
helper.log_debug('Failed to parse the challenge %s' %
str(challenge))
lines = []
raise
try:
for line in lines:
if len(line) > 0 and '=' in line:
sections = line.split('=')
line_val = self._parseJSString(sections[1])
decrypt_val = int(
eval(
str(decrypt_val) + sections[0][-1] +
str(line_val)))
except Exception as e:
helper.log_debug('Failed to find the decrypt_val from the lines')
raise
from urlparse import urlparse
path = urlparse(url).path
netloc = urlparse(url).netloc
if not netloc:
netloc = path
answer = decrypt_val + len(netloc)
url = url.rstrip('/')
query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % (
url, jschl, answer)
import urllib
if 'type="hidden" name="pass"' in challenge:
passval = re.compile('name="pass" value="(.*?)"').findall(
challenge)[0]
query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % \
(url, urllib.quote_plus(passval), jschl, answer)
import time
time.sleep(9)
helper.end("_get_cloudflare_answer")
return query