def _get_url(self, url, params=None, data=None, headers=None, cache_limit=1):
if headers is None: headers = {}
if data is not None:
if isinstance(data, basestring):
data = data
else:
data = urllib.urlencode(data, True)
if not url.startswith('http'):
url = '%s%s%s' % (self.protocol, self.BASE_URL, url)
if params: url += '?' + urllib.urlencode(params)
_created, cached_headers, html = db_connection.get_cached_url(url, data, cache_limit=cache_limit)
if html:
logger.log('Using Cached result for: %s' % (url))
result = html
res_headers = dict(cached_headers)
else:
try:
headers['Accept-Encoding'] = 'gzip'
logger.log('+++Image Scraper Call: %s, header: %s, data: %s cache_limit: %s' % (url, headers, data, cache_limit), log_utils.LOGDEBUG)
request = urllib2.Request(url, data=data, headers=headers)
response = urllib2.urlopen(request)
result = ''
while True:
data = response.read()
if not data: break
result += data
res_headers = dict(response.info().items())
if res_headers.get('content-encoding') == 'gzip':
result = utils2.ungz(result)
db_connection.cache_url(url, result, data, res_header=res_headers)
except (ssl.SSLError, socket.timeout) as e:
logger.log('Image Scraper Timeout: %s' % (url))
return {}
except urllib2.HTTPError as e:
if e.code != 404:
logger.log('HTTP Error (%s) during image scraper http get: %s' % (e, url), log_utils.LOGWARNING)
return {}
except Exception as e:
logger.log('Error (%s) during image scraper http get: %s' % (str(e), url), log_utils.LOGWARNING)
return {}
try:
if 'application/json' in res_headers.get('content-type', ''):
return_data = utils.json_loads_as_str(result)
else:
# try/except to handle older responses that might be missing headers
try: return_data = utils.json_loads_as_str(result)
except ValueError: return_data = result
except ValueError:
return_data = ''
if result:
logger.log('Invalid JSON API Response: %s - |%s|' % (url, return_data), log_utils.LOGERROR)
return return_data
def _get_url(self, url, params=None, data=None, headers=None, cache_limit=1):
if headers is None: headers = {}
if data is not None:
if isinstance(data, basestring):
data = data
else:
data = urllib.urlencode(data, True)
if not url.startswith('http'):
url = '%s%s%s' % (self.protocol, self.BASE_URL, url)
if params: url += '?' + urllib.urlencode(params)
_created, cached_headers, html = db_connection.get_cached_url(url, data, cache_limit=cache_limit)
if html:
logger.log('Using Cached result for: %s' % (url))
result = html
res_headers = dict(cached_headers)
else:
try:
headers['Accept-Encoding'] = 'gzip'
logger.log('+++Image Scraper Call: %s, header: %s, data: %s cache_limit: %s' % (url, headers, data, cache_limit), log_utils.LOGDEBUG)
request = urllib2.Request(url, data=data, headers=headers)
response = urllib2.urlopen(request)
result = ''
while True:
data = response.read()
if not data: break
result += data
res_headers = dict(response.info().items())
if res_headers.get('content-encoding') == 'gzip':
result = utils2.ungz(result)
db_connection.cache_url(url, result, data, res_header=res_headers)
except (ssl.SSLError, socket.timeout) as e:
logger.log('Image Scraper Timeout: %s' % (url))
return {}
except urllib2.HTTPError as e:
if e.code != 404:
logger.log('HTTP Error (%s) during image scraper http get: %s' % (e, url), log_utils.LOGWARNING)
return {}
except Exception as e:
logger.log('Error (%s) during image scraper http get: %s' % (str(e), url), log_utils.LOGWARNING)
return {}
try:
if 'application/json' in res_headers.get('content-type', ''):
return_data = utils.json_loads_as_str(result)
else:
# try/except to handle older responses that might be missing headers
try: return_data = utils.json_loads_as_str(result)
except ValueError: return_data = result
except ValueError:
return_data = ''
if result:
logger.log('Invalid JSON API Response: %s - |%s|' % (url, return_data), log_utils.LOGERROR)
return return_data
def __call_trakt(self,
url,
method=None,
data=None,
params=None,
auth=True,
cache_limit=.25,
cached=True):
res_headers = {}
if not cached: cache_limit = 0
if self.offline:
db_cache_limit = int(time.time()) / 60 / 60
else:
if cache_limit > 8:
db_cache_limit = cache_limit
else:
db_cache_limit = 8
json_data = json.dumps(data) if data else None
headers = {
'Content-Type': 'application/json',
'trakt-api-key': V2_API_KEY,
'trakt-api-version': 2,
'Accept-Encoding': 'gzip'
}
url = '%s%s%s' % (self.protocol, BASE_URL, url)
if params: url += '?' + urllib.urlencode(params)
db_connection = self.__get_db_connection()
created, cached_headers, cached_result = db_connection.get_cached_url(
url, json_data, db_cache_limit)
if cached_result and (self.offline or (time.time() - created) <
(60 * 60 * cache_limit)):
result = cached_result
res_headers = dict(cached_headers)
logger.log('***Using cached result for: %s' % (url),
log_utils.LOGDEBUG)
else:
auth_retry = False
while True:
try:
if auth:
headers.update(
{'Authorization': 'Bearer %s' % (self.token)})
logger.log(
'***Trakt Call: %s, header: %s, data: %s cache_limit: %s cached: %s'
% (url, headers, json_data, cache_limit, cached),
log_utils.LOGDEBUG)
request = urllib2.Request(url,
data=json_data,
headers=headers)
if method is not None:
request.get_method = lambda: method.upper()
response = urllib2.urlopen(request, timeout=self.timeout)
result = ''
while True:
data = response.read()
if not data: break
result += data
res_headers = dict(response.info().items())
if res_headers.get('content-encoding') == 'gzip':
result = utils2.ungz(result)
db_connection.cache_url(url, result, json_data,
response.info().items())
break
except (ssl.SSLError, socket.timeout) as e:
if cached_result:
result = cached_result
logger.log(
'Temporary Trakt Error (%s). Using Cached Page Instead.'
% (str(e)), log_utils.LOGWARNING)
else:
raise TransientTraktError('Temporary Trakt Error: ' +
str(e))
except urllib2.URLError as e:
if isinstance(e, urllib2.HTTPError):
if e.code in TEMP_ERRORS:
if cached_result:
result = cached_result
logger.log(
'Temporary Trakt Error (%s). Using Cached Page Instead.'
% (str(e)), log_utils.LOGWARNING)
break
else:
raise TransientTraktError(
'Temporary Trakt Error: ' + str(e))
elif e.code == 401 or e.code == 405:
# token is fine, profile is private
if e.info().getheader('X-Private-User') == 'true':
raise TraktAuthError(
'Object is No Longer Available (%s)' %
(e.code))
# auth failure retry or a token request
elif auth_retry or url.endswith('/oauth/token'):
self.token = None
kodi.set_setting('trakt_oauth_token', '')
kodi.set_setting('trakt_refresh_token', '')
raise TraktAuthError(
'Trakt Call Authentication Failed (%s)' %
(e.code))
# first try token fail, try to refresh token
else:
result = self.refresh_token(
kodi.get_setting('trakt_refresh_token'))
self.token = result['access_token']
kodi.set_setting('trakt_oauth_token',
result['access_token'])
kodi.set_setting('trakt_refresh_token',
result['refresh_token'])
auth_retry = True
elif e.code == 404:
raise TraktNotFoundError(
'Object Not Found (%s): %s' % (e.code, url))
else:
raise
elif isinstance(e.reason, socket.timeout) or isinstance(
e.reason, ssl.SSLError):
if cached_result:
result = cached_result
logger.log(
'Temporary Trakt Error (%s). Using Cached Page Instead'
% (str(e)), log_utils.LOGWARNING)
break
else:
raise TransientTraktError(
'Temporary Trakt Error: ' + str(e))
else:
raise TraktError('Trakt Error: ' + str(e))
except:
raise
try:
js_data = utils.json_loads_as_str(result)
if 'x-sort-by' in res_headers and 'x-sort-how' in res_headers:
js_data = utils2.sort_list(res_headers['x-sort-by'],
res_headers['x-sort-how'], js_data)
except ValueError:
js_data = ''
if result:
logger.log(
'Invalid JSON Trakt API Response: %s - |%s|' %
(url, js_data), log_utils.LOGERROR)
# logger.log('Trakt Response: %s' % (response), xbmc.LOGDEBUG)
return js_data
def __call_trakt(self, url, method=None, data=None, params=None, auth=True, cache_limit=.25, cached=True):
res_headers = {}
if not cached: cache_limit = 0
if self.offline:
db_cache_limit = int(time.time()) / 60 / 60
else:
if cache_limit > 8:
db_cache_limit = cache_limit
else:
db_cache_limit = 8
json_data = json.dumps(data) if data else None
headers = {'Content-Type': 'application/json', 'trakt-api-key': V2_API_KEY, 'trakt-api-version': 2, 'Accept-Encoding': 'gzip'}
url = '%s%s%s' % (self.protocol, BASE_URL, url)
if params: url += '?' + urllib.urlencode(params)
db_connection = self.__get_db_connection()
created, cached_headers, cached_result = db_connection.get_cached_url(url, json_data, db_cache_limit)
if cached_result and (self.offline or (time.time() - created) < (60 * 60 * cache_limit)):
result = cached_result
res_headers = dict(cached_headers)
logger.log('***Using cached result for: %s' % (url), log_utils.LOGDEBUG)
else:
auth_retry = False
while True:
try:
if auth: headers.update({'Authorization': 'Bearer %s' % (self.token)})
logger.log('***Trakt Call: %s, header: %s, data: %s cache_limit: %s cached: %s' % (url, headers, json_data, cache_limit, cached), log_utils.LOGDEBUG)
request = urllib2.Request(url, data=json_data, headers=headers)
if method is not None: request.get_method = lambda: method.upper()
response = urllib2.urlopen(request, timeout=self.timeout)
result = ''
while True:
data = response.read()
if not data: break
result += data
res_headers = dict(response.info().items())
if res_headers.get('content-encoding') == 'gzip':
result = utils2.ungz(result)
db_connection.cache_url(url, result, json_data, response.info().items())
break
except (ssl.SSLError, socket.timeout) as e:
if cached_result:
result = cached_result
logger.log('Temporary Trakt Error (%s). Using Cached Page Instead.' % (str(e)), log_utils.LOGWARNING)
else:
raise TransientTraktError('Temporary Trakt Error: ' + str(e))
except urllib2.URLError as e:
if isinstance(e, urllib2.HTTPError):
if e.code in TEMP_ERRORS:
if cached_result:
result = cached_result
logger.log('Temporary Trakt Error (%s). Using Cached Page Instead.' % (str(e)), log_utils.LOGWARNING)
break
else:
raise TransientTraktError('Temporary Trakt Error: ' + str(e))
elif e.code == 401 or e.code == 405:
# token is fine, profile is private
if e.info().getheader('X-Private-User') == 'true':
raise TraktAuthError('Object is No Longer Available (%s)' % (e.code))
# auth failure retry or a token request
elif auth_retry or url.endswith('/oauth/token'):
self.token = None
kodi.set_setting('trakt_oauth_token', '')
kodi.set_setting('trakt_refresh_token', '')
raise TraktAuthError('Trakt Call Authentication Failed (%s)' % (e.code))
# first try token fail, try to refresh token
else:
result = self.refresh_token(kodi.get_setting('trakt_refresh_token'))
self.token = result['access_token']
kodi.set_setting('trakt_oauth_token', result['access_token'])
kodi.set_setting('trakt_refresh_token', result['refresh_token'])
auth_retry = True
elif e.code == 404:
raise TraktNotFoundError('Object Not Found (%s): %s' % (e.code, url))
else:
raise
elif isinstance(e.reason, socket.timeout) or isinstance(e.reason, ssl.SSLError):
if cached_result:
result = cached_result
logger.log('Temporary Trakt Error (%s). Using Cached Page Instead' % (str(e)), log_utils.LOGWARNING)
break
else:
raise TransientTraktError('Temporary Trakt Error: ' + str(e))
else:
raise TraktError('Trakt Error: ' + str(e))
except:
raise
try:
js_data = utils.json_loads_as_str(result)
if 'x-sort-by' in res_headers and 'x-sort-how' in res_headers:
js_data = utils2.sort_list(res_headers['x-sort-by'], res_headers['x-sort-how'], js_data)
except ValueError:
js_data = ''
if result:
logger.log('Invalid JSON Trakt API Response: %s - |%s|' % (url, js_data), log_utils.LOGERROR)
# logger.log('Trakt Response: %s' % (response), xbmc.LOGDEBUG)
return js_data
