def get_json(self, url, cookies, referer, form_data=None):
        ajax_response, e = self.get_html(url, cookies, referer, form_data)
        import simplejson
        json = simplejson.loads(ajax_response)

        if helper.debug_dump_html():
            helper.log_debug('JSON DUMP: %s' % json)

        return json
    def get_html(self, url, cookies, referer, form_data=None):
        html, html_exception = '', None
        try:
            self.set_cookies(cookies)
            helper.log_debug('Performing a %s operation' %
                             ('POST' if form_data else 'GET'))
            if form_data:
                html = self.http_POST(url,
                                      form_data,
                                      headers={
                                          'Referer': referer
                                      }).content
            else:
                html = self.http_GET(url, headers={'Referer': referer}).content
            if html != '':
                helper.log_debug("Saving cookies")
                self.save_cookies(cookies)
            helper.log_debug("Operation complete")
        except Exception as e:
            html_exception = e

        if helper.debug_dump_html():
            helper.log_debug('HTML DUMP: %s' % html)

        return (html, html_exception)
 def _parseJSString(self, s):
     '''
         Credit to lambda - https://offshoregit.com/lambda81/
         plugin.video.genesis\resources\lib\libraries\cloudflare.py        
     '''
     try:
         offset = 1 if s[0] == '+' else 0
         val = int(
             eval(
                 s.replace('!+[]', '1').replace('!![]', '1').replace(
                     '[]', '0').replace('(', 'str(')[offset:]))
         return val
     except Exception as e:
         helper.log_debug('_parseJSString failed with exception %s' %
                          str(e))
         pass
 def __update_row(self, id, args):
     sql_update = 'INSERT OR REPLACE INTO last_visited '\
         '(id, action, value, icon, fanart, full_title, base_title, '\
         'imdb_id, tvdb_id, tmdb_id, media_type) '\
         'VALUES (%s)' % (', '.join('?' * 11))
     # Be sure to decode the names which may contain funky characters!
     full_title = args['full_title'].decode(
         'utf8') if args['full_title'] else ''
     base_title = args['base_title'].decode(
         'utf8') if args['base_title'] else ''
     data = (id, args['action'], args['value'], args['icon'],
             args['fanart'], full_title, base_title, args['imdb_id'],
             args['tvdb_id'], args['tmdb_id'], args['media_type'])
     helper.log_debug('SQL INSERT OR REPLACE: %s with params %s' %
                      (sql_update, str(data)))
     self.dbcur.execute(sql_update, data)
     self.dbcon.commit()
    def _resolve_cloudflare(self,
                            url,
                            challenge,
                            form_data={},
                            headers={},
                            compression=True):
        """
            Asks _cloudflare for an URL with the answer to overcome the 
            challenge, and then attempts the resolution.
        """
        helper.start("_resolve_cloudflare")
        from urlparse import urlparse, urlunparse
        parsed_url = urlparse(url)
        cloudflare_url = urlunparse(
            (parsed_url.scheme, parsed_url.netloc, '', '', '', ''))
        query = self._get_cloudflare_answer(cloudflare_url, challenge,
                                            form_data, headers, compression)

        # Use the cloudflare jar instead for this attempt; revert back to
        # main jar after attempt with call to update_opener()
        self._update_opener_with_cloudflare()

        try:
            helper.log_debug("Attempting to resolve the challenge")
            response = Net._fetch(self, query, form_data, headers, compression)
            helper.log_debug("Resolved the challenge, updating cookies")
            for c in self._cloudflare_jar:
                self._cj.set_cookie(c)
            self._update_opener()
        except urllib2.HTTPError as e:
            helper.log_debug(
                "Failed to resolve the cloudflare challenge with exception %s"
                % str(e))
            self._update_opener()
            pass
        helper.end('_resolve_cloudflare')
 def __get_row(self, id):
     sql_select = 'SELECT * FROM last_visited WHERE id=?'
     helper.log_debug('SQL SELECT: %s with params: %s' % (sql_select, id))
     self.dbcur.execute(sql_select, (id, ))
     matchedrow = self.dbcur.fetchone()
     return dict(matchedrow) if matchedrow else None
 def _fetch(self, url, form_data={}, headers={}, compression=True):
     '''
         A wrapper around the super's _fetch with cloudflare support
     '''
     helper.log_debug("Fetch attempt url: %s, form_data: %s, headers: %s" %
                      (url, form_data, headers))
     if not self._cloudflare:
         return Net._fetch(self, url, form_data, headers, compression)
     else:
         try:
             r = Net._fetch(self, url, form_data, headers, compression)
             helper.log_debug('Did not encounter a cloudflare challenge')
             return r
         except urllib2.HTTPError as e:
             if e.code == 503:
                 helper.log_debug('Encountered a cloudflare challenge')
                 challenge = e.read()
                 if challenge == 'The service is unavailable.':
                     helper.log_debug(
                         'Challenge says the service is unavailable')
                     raise
                 try:
                     helper.log_debug(
                         "Received a challenge, so we'll need to get around cloudflare"
                     )
                     self._resolve_cloudflare(url, challenge, form_data,
                                              headers, compression)
                     helper.log_debug(
                         "Successfully resolved cloudflare challenge, fetching real response"
                     )
                     return Net._fetch(self, url, form_data, headers,
                                       compression)
                 except urllib2.HTTPError as e:
                     helper.log_debug(
                         "Failed to set up cloudflare with exception %s" %
                         str(e))
                     raise
             else:
                 helper.log_debug('Initial attempt failed with code %d' %
                                  e.code)
                 raise
    def _get_cloudflare_answer(self,
                               url,
                               challenge,
                               form_data={},
                               headers={},
                               compression=True):
        '''
            Use the cloudflare cookie jar to overcome the cloudflare challenge.
            Returns an URL with the answer to try.

            Credit to lambda - https://offshoregit.com/lambda81/
            plugin.video.genesis\resources\lib\libraries\cloudflare.py        
        '''
        helper.start("_get_cloudflare_answer")
        if not challenge:
            helper.log_debug('Challenge is empty, re')
            raise ValueError('Challenge is empty')

        import re
        try:
            jschl = re.compile('name="jschl_vc" value="(.+?)"/>').findall(
                challenge)[0]
            init_str = re.compile('setTimeout\(function\(\){\s*.*?.*:(.*?)};'
                                  ).findall(challenge)[0]
            builder = re.compile(r"challenge-form\'\);\s*(.*)a.v").findall(
                challenge)[0]
            decrypt_val = self._parseJSString(init_str)
            lines = builder.split(';')
        except Exception as e:
            helper.log_debug('Failed to parse the challenge %s' %
                             str(challenge))
            lines = []
            raise

        try:
            for line in lines:
                if len(line) > 0 and '=' in line:
                    sections = line.split('=')
                    line_val = self._parseJSString(sections[1])
                    decrypt_val = int(
                        eval(
                            str(decrypt_val) + sections[0][-1] +
                            str(line_val)))
        except Exception as e:
            helper.log_debug('Failed to find the decrypt_val from the lines')
            raise

        from urlparse import urlparse
        path = urlparse(url).path
        netloc = urlparse(url).netloc
        if not netloc:
            netloc = path

        answer = decrypt_val + len(netloc)

        url = url.rstrip('/')
        query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % (
            url, jschl, answer)

        import urllib
        if 'type="hidden" name="pass"' in challenge:
            passval = re.compile('name="pass" value="(.*?)"').findall(
                challenge)[0]
            query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % \
                    (url, urllib.quote_plus(passval), jschl, answer)
            import time
            time.sleep(9)

        helper.end("_get_cloudflare_answer")
        return query