def _call(self, url, data=None, fmt=_urd_typeify): assert self._url, "No urd configured for this daemon" url = url.replace(' ', '%20') if data is not None: req = Request(url, json_encode(data), self._headers) else: req = Request(url) tries_left = 3 while True: try: r = urlopen(req) try: return fmt(r.read()) finally: try: r.close() except Exception: pass except HTTPError as e: if e.code in (401, 409,): raise tries_left -= 1 if not tries_left: raise print('Error %d from urd, %d tries left' % (e.code, tries_left,), file=sys.stderr) except ValueError: tries_left -= 1 if not tries_left: raise print('Bad data from urd, %d tries left' % (tries_left,), file=sys.stderr) except URLError: print('Error contacting urd', file=sys.stderr) time.sleep(4)
def get_access_token(self, verifier=None): """ After user has authorized the request token, get access token with user supplied verifier. """ try: url = self._get_oauth_url('access_token') # build request request = oauth.OAuthRequest.from_consumer_and_token( self._consumer, token=self.request_token, http_url=url, verifier=str(verifier)) request.sign_request(self._sigmethod, self._consumer, self.request_token) # send request resp = urlopen(Request(request.to_url())) # must self.access_token = oauth.OAuthToken.from_string( resp.read().decode('ascii')) #print ('Access token key: ' + str(self.access_token.key)) #print ('Access token secret: ' + str(self.access_token.secret)) return self.access_token except Exception as e: raise QWeiboError(e)
def refresh(self, refresh_token=None): """Get a new access token from the supplied refresh token https://svn.tools.ietf.org/html/draft-hammer-oauth2-00#section-4 """ endpoint = 'access_token' refresh_token = refresh_token or self.refresh_token if not refresh_token: raise ValueError("refresh_token can't be empty") args = { 'grant_type': 'refresh_token', 'client_id': self._api_key, 'refresh_token': refresh_token, } uri = urlparse.urljoin(self.BASE_URL, endpoint) body = urlencode(args) headers = { 'Content-Type': 'application/x-www-form-urlencoded', } req = Request(uri, body, headers) resp = urlopen(req) content = resp.read() if not resp.code == 200: raise Error(content) response_args = dict(parse_qsl(content)) self.access_token = response_args.get("access_token", None) self.refresh_token = response_args.get("refresh_token", None) return response_args
def get_access_token(self, verifier=None): """ After user has authorized the request token, get access token with user supplied verifier. """ try: url = self._get_oauth_url('access_token') # build request request = oauth.OAuthRequest.from_consumer_and_token( self._consumer, token=self.request_token, http_url=url, verifier=str(verifier) ) request.sign_request(self._sigmethod, self._consumer, self.request_token) # send request resp = urlopen(Request(request.to_url())) # must self.access_token = oauth.OAuthToken.from_string(resp.read().decode('ascii')) #print ('Access token key: ' + str(self.access_token.key)) #print ('Access token secret: ' + str(self.access_token.secret)) return self.access_token except Exception as e: raise QWeiboError(e)
def get_access_token(self, code): """user code to access token Get an access token from the supplied code https://svn.tools.ietf.org/html/draft-hammer-oauth2-00#section-3.5.2.2 """ if code is None: raise ValueError("Code must be set.") endpoint='access_token' params = {} if 'state' in self.params: params['state'] = self.params['state'] args = { 'grant_type': 'authorization_code', 'client_id': self._api_key, 'client_secret': self._api_secret, 'code': code, 'redirect_uri': self.callback, } args.update(params or {}) uri = urlparse.urljoin(self.BASE_URL, endpoint) body = urlencode(args) headers = { 'Content-Type': 'application/x-www-form-urlencoded', } req = Request(uri, body, headers) resp = urlopen(req) content = resp.read() if not resp.code == 200: print (resp, resp.code, content) raise Error(content) response_args = dict(parse_qsl(content)) error = response_args.get('error', None) if error is not None: msg = "%s:%s" % (error, response_args.get('error_description', '')) raise Error(msg) refresh_token = response_args.get('refresh_token', None) access_token = response_args.get('access_token', None) openid = response_args.get('openid', None) self.refresh_token = refresh_token self.access_token = access_token self.openid = openid if refresh_token is not None: response_args = self.refresh(refresh_token) return response_args
def get_access_token(self, code): """user code to access token Get an access token from the supplied code https://svn.tools.ietf.org/html/draft-hammer-oauth2-00#section-3.5.2.2 """ if code is None: raise ValueError("Code must be set.") endpoint = 'access_token' params = {} if 'state' in self.params: params['state'] = self.params['state'] args = { 'grant_type': 'authorization_code', 'client_id': self._api_key, 'client_secret': self._api_secret, 'code': code, 'redirect_uri': self.callback, } args.update(params or {}) uri = urlparse.urljoin(self.BASE_URL, endpoint) body = urlencode(args) headers = { 'Content-Type': 'application/x-www-form-urlencoded', } req = Request(uri, body, headers) resp = urlopen(req) content = resp.read() if not resp.code == 200: print(resp, resp.code, content) raise Error(content) response_args = dict(parse_qsl(content)) error = response_args.get('error', None) if error is not None: msg = "%s:%s" % (error, response_args.get('error_description', '')) raise Error(msg) refresh_token = response_args.get('refresh_token', None) access_token = response_args.get('access_token', None) openid = response_args.get('openid', None) self.refresh_token = refresh_token self.access_token = access_token self.openid = openid if refresh_token is not None: response_args = self.refresh(refresh_token) return response_args
def _url_get(self, *path, **kw): url = self.url + os.path.join('/', *path) req = urlopen(url, **kw) try: resp = req.read() finally: req.close() if PY3: resp = resp.decode('utf-8') return resp
def _get_request_token(self): try: url = self._get_oauth_url('request_token') request = oauth.OAuthRequest.from_consumer_and_token( self._consumer, http_url=url, callback=self.callback) request.sign_request(self._sigmethod, self._consumer, None) resp = urlopen(Request(request.to_url())) # must return oauth.OAuthToken.from_string(resp.read().decode('ascii')) except RuntimeError as e: raise QWeiboError(e)
def _get_request_token(self): try: url = self._get_oauth_url('request_token') request = oauth.OAuthRequest.from_consumer_and_token( self._consumer, http_url=url, callback=self.callback ) request.sign_request(self._sigmethod, self._consumer, None) resp = urlopen(Request(request.to_url())) # must return oauth.OAuthToken.from_string(resp.read().decode('ascii')) except RuntimeError as e: raise QWeiboError(e)
def resolve_remote(self, uri): """ Resolve a remote ``uri``. If called directly, does not check the store first, but after retrieving the document at the specified URI it will be saved in the store if :attr:`cache_remote` is True. .. note:: If the requests_ library is present, ``jsonschema`` will use it to request the remote ``uri``, so that the correct encoding is detected and used. If it isn't, or if the scheme of the ``uri`` is not ``http`` or ``https``, UTF-8 is assumed. Arguments: uri (str): The URI to resolve Returns: The retrieved document .. _requests: http://pypi.python.org/pypi/requests/ """ scheme = urlsplit(uri).scheme if scheme in self.handlers: result = self.handlers[scheme](uri) elif ( scheme in [u"http", u"https"] and requests and getattr(requests.Response, "json", None) is not None ): # Requests has support for detecting the correct encoding of # json over http if callable(requests.Response.json): result = requests.get(uri).json() else: result = requests.get(uri).json else: # Otherwise, pass off to urllib and assume utf-8 result = json.loads(urlopen(uri).read().decode("utf-8")) if self.cache_remote: self.store[uri] = result return result
def _next_key_and_value(self): """ Parse a single attribute type and value pair from one or more lines of LDIF data Returns attr_type (text) and attr_value (bytes) """ # Reading new attribute line unfolded_line = self._unfold_lines() # Ignore comments which can also be folded while unfolded_line and unfolded_line[0]=='#': unfolded_line = self._unfold_lines() if not unfolded_line: return None,None if unfolded_line=='-': return '-',None try: colon_pos = unfolded_line.index(':') except ValueError as e: raise ValueError('no value-spec in %s' % (repr(unfolded_line))) attr_type = unfolded_line[0:colon_pos] # if needed attribute value is BASE64 decoded value_spec = unfolded_line[colon_pos:colon_pos+2] if value_spec==': ': attr_value = unfolded_line[colon_pos+2:].lstrip() # All values should be valid ascii; we support UTF-8 as a # non-official, backwards compatibility layer. attr_value = attr_value.encode('utf-8') elif value_spec=='::': # attribute value needs base64-decoding # base64 makes sens only for ascii attr_value = unfolded_line[colon_pos+2:] attr_value = attr_value.encode('ascii') attr_value = self._b64decode(attr_value) elif value_spec==':<': # fetch attribute value from URL url = unfolded_line[colon_pos+2:].strip() attr_value = None if self._process_url_schemes: u = urlparse(url) if u[0] in self._process_url_schemes: attr_value = urlopen(url).read() else: # All values should be valid ascii; we support UTF-8 as a # non-official, backwards compatibility layer. attr_value = unfolded_line[colon_pos+1:].encode('utf-8') return attr_type,attr_value
def resolve_remote(self, uri): """ Resolve a remote ``uri``. Does not check the store first, but stores the retrieved document in the store if :attr:`RefResolver.cache_remote` is True. .. note:: If the requests_ library is present, ``jsonschema`` will use it to request the remote ``uri``, so that the correct encoding is detected and used. If it isn't, or if the scheme of the ``uri`` is not ``http`` or ``https``, UTF-8 is assumed. :argument str uri: the URI to resolve :returns: the retrieved document .. _requests: http://pypi.python.org/pypi/requests/ """ scheme = urlsplit(uri).scheme if scheme in self.handlers: result = self.handlers[scheme](uri) elif ( scheme in ["http", "https"] and requests and getattr(requests.Response, "json", None) is not None ): # Requests has support for detecting the correct encoding of # json over http if callable(requests.Response.json): result = requests.get(uri).json() else: result = requests.get(uri).json else: # Otherwise, pass off to urllib and assume utf-8 result = json.loads(urlopen(uri).read().decode("utf-8")) if self.cache_remote: self.store[uri] = result return result
def __call__(self, *args, **kargs): if len(args): raise TypeError('Only keyword arguments are allowed') form = _MultiPartForm() for (k, v) in kargs.items(): if isinstance(v, File): form.add_file(k, v.get_filename(), v.content) url = self._urlbase for k, v in self._mkarg(kargs).items(): form.add_field(k, v) body = form.bytes request = Request(url, data=body) request.add_header('Content-type', form.get_content_type()) request.add_header('Content-length', str(len(body))) self._api.update_request(request) retry = self._api.max_retries while True: retry -= 1 try: ret = urlopen(request, timeout=self._api.timeout).read() break except HTTPError as e: raise APIError(e.code, url, e.read()) except (socket.error, URLError) as e: if retry < 0: raise e _print_debug('caught error: {}; retrying'.format(e)) time.sleep(self._api.retry_delay) if self._api.decode_result: try: ret = json.loads(ret, object_hook=ObjectDict) except: raise APIError(-1, url, 'json decode error, value={0!r}'.format(ret)) return ret
def _json_file(path): return json.loads(urlopen(path).read().decode("utf-8"))
def execute(self): # Build the request URL url = self.api_root + self.path #if self.api.source is not None: # self.parameters.setdefault('source',self.api.source) if len(self.parameters): if self.method == 'GET': url = '%s?%s' % (url, urlencode(self.parameters)) else: self.headers.setdefault("User-Agent", "pyqqweibo") if self.post_data is None: self.headers.setdefault("Accept", "text/html") self.headers.setdefault("Content-Type", "application/x-www-form-urlencoded") # asure in bytes format self.post_data = urlencode(self.parameters).encode('ascii') # Query the cache if one is available # and this request uses a GET method. if self.api.cache and self.method == 'GET': cache_result = self.api.cache.get(url) # if cache result found and not expired, return it if cache_result: # must restore api reference if isinstance(cache_result, list): for result in cache_result: result._api = self.api else: cache_result._api = self.api return cache_result #urllib.urlencode(self.parameters) # Continue attempting request until successful # or maximum number of retries is reached. sTime = time.time() retries_performed = 0 while retries_performed < self.retry_count + 1: # Open connection # FIXME: add timeout # Apply authentication if self.require_auth and self.api.auth: url_full = self.api.auth.get_authed_url( self.scheme + self.host + url, self.method, self.headers, self.parameters ) else: # this brunch is never accoured url_full = self.api.auth.get_signed_url( self.scheme + self.host + url, self.method, self.headers, self.parameters ) try: if self.method == 'POST': req = Request(url_full, data=self.post_data, headers=self.headers) else: req = Request(url_full) resp = urlopen(req) except Exception as e: raise QWeiboError("Failed to request %s headers=%s %s" % \ (url, self.headers, e)) # Exit request loop if non-retry error code if self.retry_errors: if resp.code not in self.retry_errors: break else: if resp.code == 200: break # Sleep before retrying request again time.sleep(self.retry_delay) retries_performed += 1 # If an error was returned, throw an exception body = resp.read() self.api.last_response = resp if self.api.log is not None: requestUrl = "URL:http://" + self.host + url eTime = '%.0f' % ((time.time() - sTime) * 1000) postData = "" if self.post_data is not None: postData = ",post:" + self.post_data[:500] self.api.log.debug("%s, time: %s, %s result: %s" % (requestUrl, eTime, postData, body)) retcode = 0 errcode = 0 # for py3k, ^_^ if not hasattr(body, 'encode'): body = str(body, 'utf-8') if self.api.parser.payload_format == 'json': try: # BUG: API BUG, refer api.doc.rst if body.endswith('out of memery'): body = body[:body.rfind('}')+1] json = self.api.parser.parse_error(self, body) retcode = json.get('ret', 0) msg = json.get('msg', '') # only in some post request errcode = json.get('errcode', 0) except ValueError as e: retcode = -1 msg = "Bad json format (%s)" % e finally: if retcode + errcode != 0: raise QWeiboError("Response error: %s. (ret=%s, errcode=%s)" % \ (msg, retcode, errcode)) # Parse the response payload result = self.api.parser.parse(self, body) # Store result into cache if one is available. if self.api.cache and self.method == 'GET' and result: self.api.cache.store(url, result) return result