def _call(self, url, data=None, fmt=_urd_typeify):
assert self._url, "No urd configured for this daemon"
url = url.replace(' ', '%20')
if data is not None:
req = Request(url, json_encode(data), self._headers)
else:
req = Request(url)
tries_left = 3
while True:
try:
r = urlopen(req)
try:
return fmt(r.read())
finally:
try:
r.close()
except Exception:
pass
except HTTPError as e:
if e.code in (401, 409,):
raise
tries_left -= 1
if not tries_left:
raise
print('Error %d from urd, %d tries left' % (e.code, tries_left,), file=sys.stderr)
except ValueError:
tries_left -= 1
if not tries_left:
raise
print('Bad data from urd, %d tries left' % (tries_left,), file=sys.stderr)
except URLError:
print('Error contacting urd', file=sys.stderr)
time.sleep(4)
def get_access_token(self, verifier=None):
"""
After user has authorized the request token, get access token
with user supplied verifier.
"""
try:
url = self._get_oauth_url('access_token')
# build request
request = oauth.OAuthRequest.from_consumer_and_token(
self._consumer,
token=self.request_token,
http_url=url,
verifier=str(verifier))
request.sign_request(self._sigmethod, self._consumer,
self.request_token)
# send request
resp = urlopen(Request(request.to_url())) # must
self.access_token = oauth.OAuthToken.from_string(
resp.read().decode('ascii'))
#print ('Access token key: ' + str(self.access_token.key))
#print ('Access token secret: ' + str(self.access_token.secret))
return self.access_token
except Exception as e:
raise QWeiboError(e)
def refresh(self, refresh_token=None):
"""Get a new access token from the supplied refresh token
https://svn.tools.ietf.org/html/draft-hammer-oauth2-00#section-4
"""
endpoint = 'access_token'
refresh_token = refresh_token or self.refresh_token
if not refresh_token:
raise ValueError("refresh_token can't be empty")
args = {
'grant_type': 'refresh_token',
'client_id': self._api_key,
'refresh_token': refresh_token,
}
uri = urlparse.urljoin(self.BASE_URL, endpoint)
body = urlencode(args)
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
req = Request(uri, body, headers)
resp = urlopen(req)
content = resp.read()
if not resp.code == 200:
raise Error(content)
response_args = dict(parse_qsl(content))
self.access_token = response_args.get("access_token", None)
self.refresh_token = response_args.get("refresh_token", None)
return response_args
def get_access_token(self, verifier=None):
"""
After user has authorized the request token, get access token
with user supplied verifier.
"""
try:
url = self._get_oauth_url('access_token')
# build request
request = oauth.OAuthRequest.from_consumer_and_token(
self._consumer,
token=self.request_token, http_url=url,
verifier=str(verifier)
)
request.sign_request(self._sigmethod, self._consumer, self.request_token)
# send request
resp = urlopen(Request(request.to_url())) # must
self.access_token = oauth.OAuthToken.from_string(resp.read().decode('ascii'))
#print ('Access token key: ' + str(self.access_token.key))
#print ('Access token secret: ' + str(self.access_token.secret))
return self.access_token
except Exception as e:
raise QWeiboError(e)
def refresh(self, refresh_token=None):
"""Get a new access token from the supplied refresh token
https://svn.tools.ietf.org/html/draft-hammer-oauth2-00#section-4
"""
endpoint = 'access_token'
refresh_token = refresh_token or self.refresh_token
if not refresh_token:
raise ValueError("refresh_token can't be empty")
args = {
'grant_type': 'refresh_token',
'client_id': self._api_key,
'refresh_token': refresh_token,
}
uri = urlparse.urljoin(self.BASE_URL, endpoint)
body = urlencode(args)
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
req = Request(uri, body, headers)
resp = urlopen(req)
content = resp.read()
if not resp.code == 200:
raise Error(content)
response_args = dict(parse_qsl(content))
self.access_token = response_args.get("access_token", None)
self.refresh_token = response_args.get("refresh_token", None)
return response_args
def get_access_token(self, code):
"""user code to access token
Get an access token from the supplied code
https://svn.tools.ietf.org/html/draft-hammer-oauth2-00#section-3.5.2.2
"""
if code is None:
raise ValueError("Code must be set.")
endpoint='access_token'
params = {}
if 'state' in self.params:
params['state'] = self.params['state']
args = {
'grant_type': 'authorization_code',
'client_id': self._api_key,
'client_secret': self._api_secret,
'code': code,
'redirect_uri': self.callback,
}
args.update(params or {})
uri = urlparse.urljoin(self.BASE_URL, endpoint)
body = urlencode(args)
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
req = Request(uri, body, headers)
resp = urlopen(req)
content = resp.read()
if not resp.code == 200:
print (resp, resp.code, content)
raise Error(content)
response_args = dict(parse_qsl(content))
error = response_args.get('error', None)
if error is not None:
msg = "%s:%s" % (error,
response_args.get('error_description', ''))
raise Error(msg)
refresh_token = response_args.get('refresh_token', None)
access_token = response_args.get('access_token', None)
openid = response_args.get('openid', None)
self.refresh_token = refresh_token
self.access_token = access_token
self.openid = openid
if refresh_token is not None:
response_args = self.refresh(refresh_token)
return response_args
def get_access_token(self, code):
"""user code to access token
Get an access token from the supplied code
https://svn.tools.ietf.org/html/draft-hammer-oauth2-00#section-3.5.2.2
"""
if code is None:
raise ValueError("Code must be set.")
endpoint = 'access_token'
params = {}
if 'state' in self.params:
params['state'] = self.params['state']
args = {
'grant_type': 'authorization_code',
'client_id': self._api_key,
'client_secret': self._api_secret,
'code': code,
'redirect_uri': self.callback,
}
args.update(params or {})
uri = urlparse.urljoin(self.BASE_URL, endpoint)
body = urlencode(args)
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
req = Request(uri, body, headers)
resp = urlopen(req)
content = resp.read()
if not resp.code == 200:
print(resp, resp.code, content)
raise Error(content)
response_args = dict(parse_qsl(content))
error = response_args.get('error', None)
if error is not None:
msg = "%s:%s" % (error, response_args.get('error_description', ''))
raise Error(msg)
refresh_token = response_args.get('refresh_token', None)
access_token = response_args.get('access_token', None)
openid = response_args.get('openid', None)
self.refresh_token = refresh_token
self.access_token = access_token
self.openid = openid
if refresh_token is not None:
response_args = self.refresh(refresh_token)
return response_args
def _url_get(self, *path, **kw):
url = self.url + os.path.join('/', *path)
req = urlopen(url, **kw)
try:
resp = req.read()
finally:
req.close()
if PY3:
resp = resp.decode('utf-8')
return resp
def _get_request_token(self):
try:
url = self._get_oauth_url('request_token')
request = oauth.OAuthRequest.from_consumer_and_token(
self._consumer, http_url=url, callback=self.callback)
request.sign_request(self._sigmethod, self._consumer, None)
resp = urlopen(Request(request.to_url())) # must
return oauth.OAuthToken.from_string(resp.read().decode('ascii'))
except RuntimeError as e:
raise QWeiboError(e)
def _get_request_token(self):
try:
url = self._get_oauth_url('request_token')
request = oauth.OAuthRequest.from_consumer_and_token(
self._consumer, http_url=url, callback=self.callback
)
request.sign_request(self._sigmethod, self._consumer, None)
resp = urlopen(Request(request.to_url())) # must
return oauth.OAuthToken.from_string(resp.read().decode('ascii'))
except RuntimeError as e:
raise QWeiboError(e)
def resolve_remote(self, uri):
"""
Resolve a remote ``uri``.
If called directly, does not check the store first, but after
retrieving the document at the specified URI it will be saved in
the store if :attr:`cache_remote` is True.
.. note::
If the requests_ library is present, ``jsonschema`` will use it to
request the remote ``uri``, so that the correct encoding is
detected and used.
If it isn't, or if the scheme of the ``uri`` is not ``http`` or
``https``, UTF-8 is assumed.
Arguments:
uri (str):
The URI to resolve
Returns:
The retrieved document
.. _requests: http://pypi.python.org/pypi/requests/
"""
scheme = urlsplit(uri).scheme
if scheme in self.handlers:
result = self.handlers[scheme](uri)
elif (
scheme in [u"http", u"https"] and
requests and
getattr(requests.Response, "json", None) is not None
):
# Requests has support for detecting the correct encoding of
# json over http
if callable(requests.Response.json):
result = requests.get(uri).json()
else:
result = requests.get(uri).json
else:
# Otherwise, pass off to urllib and assume utf-8
result = json.loads(urlopen(uri).read().decode("utf-8"))
if self.cache_remote:
self.store[uri] = result
return result
def _next_key_and_value(self):
"""
Parse a single attribute type and value pair from one or
more lines of LDIF data
Returns attr_type (text) and attr_value (bytes)
"""
# Reading new attribute line
unfolded_line = self._unfold_lines()
# Ignore comments which can also be folded
while unfolded_line and unfolded_line[0]=='#':
unfolded_line = self._unfold_lines()
if not unfolded_line:
return None,None
if unfolded_line=='-':
return '-',None
try:
colon_pos = unfolded_line.index(':')
except ValueError as e:
raise ValueError('no value-spec in %s' % (repr(unfolded_line)))
attr_type = unfolded_line[0:colon_pos]
# if needed attribute value is BASE64 decoded
value_spec = unfolded_line[colon_pos:colon_pos+2]
if value_spec==': ':
attr_value = unfolded_line[colon_pos+2:].lstrip()
# All values should be valid ascii; we support UTF-8 as a
# non-official, backwards compatibility layer.
attr_value = attr_value.encode('utf-8')
elif value_spec=='::':
# attribute value needs base64-decoding
# base64 makes sens only for ascii
attr_value = unfolded_line[colon_pos+2:]
attr_value = attr_value.encode('ascii')
attr_value = self._b64decode(attr_value)
elif value_spec==':<':
# fetch attribute value from URL
url = unfolded_line[colon_pos+2:].strip()
attr_value = None
if self._process_url_schemes:
u = urlparse(url)
if u[0] in self._process_url_schemes:
attr_value = urlopen(url).read()
else:
# All values should be valid ascii; we support UTF-8 as a
# non-official, backwards compatibility layer.
attr_value = unfolded_line[colon_pos+1:].encode('utf-8')
return attr_type,attr_value
def resolve_remote(self, uri):
"""
Resolve a remote ``uri``.
Does not check the store first, but stores the retrieved document in
the store if :attr:`RefResolver.cache_remote` is True.
.. note::
If the requests_ library is present, ``jsonschema`` will use it to
request the remote ``uri``, so that the correct encoding is
detected and used.
If it isn't, or if the scheme of the ``uri`` is not ``http`` or
``https``, UTF-8 is assumed.
:argument str uri: the URI to resolve
:returns: the retrieved document
.. _requests: http://pypi.python.org/pypi/requests/
"""
scheme = urlsplit(uri).scheme
if scheme in self.handlers:
result = self.handlers[scheme](uri)
elif (
scheme in ["http", "https"] and
requests and
getattr(requests.Response, "json", None) is not None
):
# Requests has support for detecting the correct encoding of
# json over http
if callable(requests.Response.json):
result = requests.get(uri).json()
else:
result = requests.get(uri).json
else:
# Otherwise, pass off to urllib and assume utf-8
result = json.loads(urlopen(uri).read().decode("utf-8"))
if self.cache_remote:
self.store[uri] = result
return result
def __call__(self, *args, **kargs):
if len(args):
raise TypeError('Only keyword arguments are allowed')
form = _MultiPartForm()
for (k, v) in kargs.items():
if isinstance(v, File):
form.add_file(k, v.get_filename(), v.content)
url = self._urlbase
for k, v in self._mkarg(kargs).items():
form.add_field(k, v)
body = form.bytes
request = Request(url, data=body)
request.add_header('Content-type', form.get_content_type())
request.add_header('Content-length', str(len(body)))
self._api.update_request(request)
retry = self._api.max_retries
while True:
retry -= 1
try:
ret = urlopen(request, timeout=self._api.timeout).read()
break
except HTTPError as e:
raise APIError(e.code, url, e.read())
except (socket.error, URLError) as e:
if retry < 0:
raise e
_print_debug('caught error: {}; retrying'.format(e))
time.sleep(self._api.retry_delay)
if self._api.decode_result:
try:
ret = json.loads(ret, object_hook=ObjectDict)
except:
raise APIError(-1, url,
'json decode error, value={0!r}'.format(ret))
return ret
def _json_file(path):
return json.loads(urlopen(path).read().decode("utf-8"))
def execute(self):
# Build the request URL
url = self.api_root + self.path
#if self.api.source is not None:
# self.parameters.setdefault('source',self.api.source)
if len(self.parameters):
if self.method == 'GET':
url = '%s?%s' % (url, urlencode(self.parameters))
else:
self.headers.setdefault("User-Agent", "pyqqweibo")
if self.post_data is None:
self.headers.setdefault("Accept", "text/html")
self.headers.setdefault("Content-Type", "application/x-www-form-urlencoded")
# asure in bytes format
self.post_data = urlencode(self.parameters).encode('ascii')
# Query the cache if one is available
# and this request uses a GET method.
if self.api.cache and self.method == 'GET':
cache_result = self.api.cache.get(url)
# if cache result found and not expired, return it
if cache_result:
# must restore api reference
if isinstance(cache_result, list):
for result in cache_result:
result._api = self.api
else:
cache_result._api = self.api
return cache_result
#urllib.urlencode(self.parameters)
# Continue attempting request until successful
# or maximum number of retries is reached.
sTime = time.time()
retries_performed = 0
while retries_performed < self.retry_count + 1:
# Open connection
# FIXME: add timeout
# Apply authentication
if self.require_auth and self.api.auth:
url_full = self.api.auth.get_authed_url(
self.scheme + self.host + url,
self.method, self.headers, self.parameters
)
else: # this brunch is never accoured
url_full = self.api.auth.get_signed_url(
self.scheme + self.host + url,
self.method, self.headers, self.parameters
)
try:
if self.method == 'POST':
req = Request(url_full, data=self.post_data, headers=self.headers)
else:
req = Request(url_full)
resp = urlopen(req)
except Exception as e:
raise QWeiboError("Failed to request %s headers=%s %s" % \
(url, self.headers, e))
# Exit request loop if non-retry error code
if self.retry_errors:
if resp.code not in self.retry_errors:
break
else:
if resp.code == 200:
break
# Sleep before retrying request again
time.sleep(self.retry_delay)
retries_performed += 1
# If an error was returned, throw an exception
body = resp.read()
self.api.last_response = resp
if self.api.log is not None:
requestUrl = "URL:http://" + self.host + url
eTime = '%.0f' % ((time.time() - sTime) * 1000)
postData = ""
if self.post_data is not None:
postData = ",post:" + self.post_data[:500]
self.api.log.debug("%s, time: %s, %s result: %s" % (requestUrl, eTime, postData, body))
retcode = 0
errcode = 0
# for py3k, ^_^
if not hasattr(body, 'encode'):
body = str(body, 'utf-8')
if self.api.parser.payload_format == 'json':
try:
# BUG: API BUG, refer api.doc.rst
if body.endswith('out of memery'):
body = body[:body.rfind('}')+1]
json = self.api.parser.parse_error(self, body)
retcode = json.get('ret', 0)
msg = json.get('msg', '')
# only in some post request
errcode = json.get('errcode', 0)
except ValueError as e:
retcode = -1
msg = "Bad json format (%s)" % e
finally:
if retcode + errcode != 0:
raise QWeiboError("Response error: %s. (ret=%s, errcode=%s)" % \
(msg, retcode, errcode))
# Parse the response payload
result = self.api.parser.parse(self, body)
# Store result into cache if one is available.
if self.api.cache and self.method == 'GET' and result:
self.api.cache.store(url, result)
return result
