def _get_url(self, url, params=None, data=None, headers=None, cache_limit=1): if headers is None: headers = {} if data is not None: if isinstance(data, basestring): data = data else: data = urllib.urlencode(data, True) if not url.startswith('http'): url = '%s%s%s' % (self.protocol, self.BASE_URL, url) if params: url += '?' + urllib.urlencode(params) _created, cached_headers, html = db_connection.get_cached_url(url, data, cache_limit=cache_limit) if html: logger.log('Using Cached result for: %s' % (url)) result = html res_headers = dict(cached_headers) else: try: headers['Accept-Encoding'] = 'gzip' logger.log('+++Image Scraper Call: %s, header: %s, data: %s cache_limit: %s' % (url, headers, data, cache_limit), log_utils.LOGDEBUG) request = urllib2.Request(url, data=data, headers=headers) response = urllib2.urlopen(request) result = '' while True: data = response.read() if not data: break result += data res_headers = dict(response.info().items()) if res_headers.get('content-encoding') == 'gzip': result = utils2.ungz(result) db_connection.cache_url(url, result, data, res_header=res_headers) except (ssl.SSLError, socket.timeout) as e: logger.log('Image Scraper Timeout: %s' % (url)) return {} except urllib2.HTTPError as e: if e.code != 404: logger.log('HTTP Error (%s) during image scraper http get: %s' % (e, url), log_utils.LOGWARNING) return {} except Exception as e: logger.log('Error (%s) during image scraper http get: %s' % (str(e), url), log_utils.LOGWARNING) return {} try: if 'application/json' in res_headers.get('content-type', ''): return_data = utils.json_loads_as_str(result) else: # try/except to handle older responses that might be missing headers try: return_data = utils.json_loads_as_str(result) except ValueError: return_data = result except ValueError: return_data = '' if result: logger.log('Invalid JSON API Response: %s - |%s|' % (url, return_data), log_utils.LOGERROR) return return_data
def __call_trakt(self, url, method=None, data=None, params=None, auth=True, cache_limit=.25, cached=True): res_headers = {} if not cached: cache_limit = 0 if self.offline: db_cache_limit = int(time.time()) / 60 / 60 else: if cache_limit > 8: db_cache_limit = cache_limit else: db_cache_limit = 8 json_data = json.dumps(data) if data else None headers = { 'Content-Type': 'application/json', 'trakt-api-key': V2_API_KEY, 'trakt-api-version': 2, 'Accept-Encoding': 'gzip' } url = '%s%s%s' % (self.protocol, BASE_URL, url) if params: url += '?' + urllib.urlencode(params) db_connection = self.__get_db_connection() created, cached_headers, cached_result = db_connection.get_cached_url( url, json_data, db_cache_limit) if cached_result and (self.offline or (time.time() - created) < (60 * 60 * cache_limit)): result = cached_result res_headers = dict(cached_headers) logger.log('***Using cached result for: %s' % (url), log_utils.LOGDEBUG) else: auth_retry = False while True: try: if auth: headers.update( {'Authorization': 'Bearer %s' % (self.token)}) logger.log( '***Trakt Call: %s, header: %s, data: %s cache_limit: %s cached: %s' % (url, headers, json_data, cache_limit, cached), log_utils.LOGDEBUG) request = urllib2.Request(url, data=json_data, headers=headers) if method is not None: request.get_method = lambda: method.upper() response = urllib2.urlopen(request, timeout=self.timeout) result = '' while True: data = response.read() if not data: break result += data res_headers = dict(response.info().items()) if res_headers.get('content-encoding') == 'gzip': result = utils2.ungz(result) db_connection.cache_url(url, result, json_data, response.info().items()) break except (ssl.SSLError, socket.timeout) as e: if cached_result: result = cached_result logger.log( 'Temporary Trakt Error (%s). Using Cached Page Instead.' % (str(e)), log_utils.LOGWARNING) else: raise TransientTraktError('Temporary Trakt Error: ' + str(e)) except urllib2.URLError as e: if isinstance(e, urllib2.HTTPError): if e.code in TEMP_ERRORS: if cached_result: result = cached_result logger.log( 'Temporary Trakt Error (%s). Using Cached Page Instead.' % (str(e)), log_utils.LOGWARNING) break else: raise TransientTraktError( 'Temporary Trakt Error: ' + str(e)) elif e.code == 401 or e.code == 405: # token is fine, profile is private if e.info().getheader('X-Private-User') == 'true': raise TraktAuthError( 'Object is No Longer Available (%s)' % (e.code)) # auth failure retry or a token request elif auth_retry or url.endswith('/oauth/token'): self.token = None kodi.set_setting('trakt_oauth_token', '') kodi.set_setting('trakt_refresh_token', '') raise TraktAuthError( 'Trakt Call Authentication Failed (%s)' % (e.code)) # first try token fail, try to refresh token else: result = self.refresh_token( kodi.get_setting('trakt_refresh_token')) self.token = result['access_token'] kodi.set_setting('trakt_oauth_token', result['access_token']) kodi.set_setting('trakt_refresh_token', result['refresh_token']) auth_retry = True elif e.code == 404: raise TraktNotFoundError( 'Object Not Found (%s): %s' % (e.code, url)) else: raise elif isinstance(e.reason, socket.timeout) or isinstance( e.reason, ssl.SSLError): if cached_result: result = cached_result logger.log( 'Temporary Trakt Error (%s). Using Cached Page Instead' % (str(e)), log_utils.LOGWARNING) break else: raise TransientTraktError( 'Temporary Trakt Error: ' + str(e)) else: raise TraktError('Trakt Error: ' + str(e)) except: raise try: js_data = utils.json_loads_as_str(result) if 'x-sort-by' in res_headers and 'x-sort-how' in res_headers: js_data = utils2.sort_list(res_headers['x-sort-by'], res_headers['x-sort-how'], js_data) except ValueError: js_data = '' if result: logger.log( 'Invalid JSON Trakt API Response: %s - |%s|' % (url, js_data), log_utils.LOGERROR) # logger.log('Trakt Response: %s' % (response), xbmc.LOGDEBUG) return js_data
def __call_trakt(self, url, method=None, data=None, params=None, auth=True, cache_limit=.25, cached=True): res_headers = {} if not cached: cache_limit = 0 if self.offline: db_cache_limit = int(time.time()) / 60 / 60 else: if cache_limit > 8: db_cache_limit = cache_limit else: db_cache_limit = 8 json_data = json.dumps(data) if data else None headers = {'Content-Type': 'application/json', 'trakt-api-key': V2_API_KEY, 'trakt-api-version': 2, 'Accept-Encoding': 'gzip'} url = '%s%s%s' % (self.protocol, BASE_URL, url) if params: url += '?' + urllib.urlencode(params) db_connection = self.__get_db_connection() created, cached_headers, cached_result = db_connection.get_cached_url(url, json_data, db_cache_limit) if cached_result and (self.offline or (time.time() - created) < (60 * 60 * cache_limit)): result = cached_result res_headers = dict(cached_headers) logger.log('***Using cached result for: %s' % (url), log_utils.LOGDEBUG) else: auth_retry = False while True: try: if auth: headers.update({'Authorization': 'Bearer %s' % (self.token)}) logger.log('***Trakt Call: %s, header: %s, data: %s cache_limit: %s cached: %s' % (url, headers, json_data, cache_limit, cached), log_utils.LOGDEBUG) request = urllib2.Request(url, data=json_data, headers=headers) if method is not None: request.get_method = lambda: method.upper() response = urllib2.urlopen(request, timeout=self.timeout) result = '' while True: data = response.read() if not data: break result += data res_headers = dict(response.info().items()) if res_headers.get('content-encoding') == 'gzip': result = utils2.ungz(result) db_connection.cache_url(url, result, json_data, response.info().items()) break except (ssl.SSLError, socket.timeout) as e: if cached_result: result = cached_result logger.log('Temporary Trakt Error (%s). Using Cached Page Instead.' % (str(e)), log_utils.LOGWARNING) else: raise TransientTraktError('Temporary Trakt Error: ' + str(e)) except urllib2.URLError as e: if isinstance(e, urllib2.HTTPError): if e.code in TEMP_ERRORS: if cached_result: result = cached_result logger.log('Temporary Trakt Error (%s). Using Cached Page Instead.' % (str(e)), log_utils.LOGWARNING) break else: raise TransientTraktError('Temporary Trakt Error: ' + str(e)) elif e.code == 401 or e.code == 405: # token is fine, profile is private if e.info().getheader('X-Private-User') == 'true': raise TraktAuthError('Object is No Longer Available (%s)' % (e.code)) # auth failure retry or a token request elif auth_retry or url.endswith('/oauth/token'): self.token = None kodi.set_setting('trakt_oauth_token', '') kodi.set_setting('trakt_refresh_token', '') raise TraktAuthError('Trakt Call Authentication Failed (%s)' % (e.code)) # first try token fail, try to refresh token else: result = self.refresh_token(kodi.get_setting('trakt_refresh_token')) self.token = result['access_token'] kodi.set_setting('trakt_oauth_token', result['access_token']) kodi.set_setting('trakt_refresh_token', result['refresh_token']) auth_retry = True elif e.code == 404: raise TraktNotFoundError('Object Not Found (%s): %s' % (e.code, url)) else: raise elif isinstance(e.reason, socket.timeout) or isinstance(e.reason, ssl.SSLError): if cached_result: result = cached_result logger.log('Temporary Trakt Error (%s). Using Cached Page Instead' % (str(e)), log_utils.LOGWARNING) break else: raise TransientTraktError('Temporary Trakt Error: ' + str(e)) else: raise TraktError('Trakt Error: ' + str(e)) except: raise try: js_data = utils.json_loads_as_str(result) if 'x-sort-by' in res_headers and 'x-sort-how' in res_headers: js_data = utils2.sort_list(res_headers['x-sort-by'], res_headers['x-sort-how'], js_data) except ValueError: js_data = '' if result: logger.log('Invalid JSON Trakt API Response: %s - |%s|' % (url, js_data), log_utils.LOGERROR) # logger.log('Trakt Response: %s' % (response), xbmc.LOGDEBUG) return js_data