def update_qs(uri, new_params, overwrite=True): """Helper function for updating query string values. Similar to calling update on a dictionary except we modify the query string of the uri instead of another dictionary. :arg uri: URI to modify :arg new_params: Dict of new query parameters to add. :kwarg overwrite: If True (default), any old query parameter with the same name as a new query parameter will be overwritten. If False, the new query parameters will be appended to a list with the old parameters at the start of the list. :returns: URI with the new parameters added. """ loc = list(urlparse(uri)) query_dict = parse_qs(loc[4]) if overwrite: # Overwrite any existing values with the new values query_dict.update(new_params) else: # Add new values in addition to the existing parameters # For every new entry for key in new_params: # If the entry already is present if key in query_dict: if isiterable(new_params[key]): # If the new entry is a non-string iterable try: # Try to add the new values to the existing entry query_dict[key].extend(new_params[key]) except AttributeError: # Existing value wasn't a list, make it one query_dict[key] = [query_dict[key], new_params[key]] else: # The new entry is a scalar, so try to append it try: query_dict[key].append(new_params[key]) except AttributeError: # Existing value wasn't a list, make it one query_dict[key] = [query_dict[key], new_params[key]] else: # No previous entry, just set to the new entry query_dict[key] = new_params[key] # seems that we have to sanitize a bit here query_list = [] for key, value in query_dict.items(): if isiterable(value): for item in value: query_list.append((key, item)) continue query_list.append((key, value)) loc[4] = urlencode(query_list) return urlunparse(loc)
def update_qs(uri, new_params, overwrite=True): '''Helper function for updating query string values. Similar to calling update on a dictionary except we modify the query string of the uri instead of another dictionary. :arg uri: URI to modify :arg new_params: Dict of new query parameters to add. :kwarg overwrite: If True (default), any old query parameter with the same name as a new query parameter will be overwritten. If False, the new query parameters will be appended to a list with the old parameters at the start of the list. :returns: URI with the new parameters added. ''' loc = list(urlparse(uri)) query_dict = parse_qs(loc[4]) if overwrite: # Overwrite any existing values with the new values query_dict.update(new_params) else: # Add new values in addition to the existing parameters # For every new entry for key in new_params: # If the entry already is present if key in query_dict: if isiterable(new_params[key]): # If the new entry is a non-string iterable try: # Try to add the new values to the existing entry query_dict[key].extend(new_params[key]) except AttributeError: # Existing value wasn't a list, make it one query_dict[key] = [query_dict[key], new_params[key]] else: # The new entry is a scalar, so try to append it try: query_dict[key].append(new_params[key]) except AttributeError: # Existing value wasn't a list, make it one query_dict[key] = [query_dict[key], new_params[key]] else: # No previous entry, just set to the new entry query_dict[key] = new_params[key] # seems that we have to sanitize a bit here query_list = [] for key, value in query_dict.items(): if isiterable(value): for item in value: query_list.append((key, item)) continue query_list.append((key, value)) loc[4] = urlencode(query_list) return urlunparse(loc)
def test_isiterable(self): for item in self.iterable_data: tools.ok_(iterutils.isiterable(item) == True) for item in self.non_iterable_data: tools.ok_(iterutils.isiterable(item) == False) # strings tools.ok_(iterutils.isiterable('a', include_string=True) == True) tools.ok_(iterutils.isiterable('a', include_string=False) == False) tools.ok_(iterutils.isiterable('a') == False)
def test_isiterable(self): for item in self.iterable_data: tools.ok_(iterutils.isiterable(item) == True) for item in self.non_iterable_data: tools.ok_(iterutils.isiterable(item) == False) # strings tools.ok_(iterutils.isiterable('a', include_string=True) == True) tools.ok_(iterutils.isiterable('a', include_string=False) == False) tools.ok_(iterutils.isiterable('a') == False) tools.ok_(iterutils.isiterable(u'a', include_string=True) == True) tools.ok_(iterutils.isiterable(u'a', include_string=False) == False) tools.ok_(iterutils.isiterable(u'a') == False)
def send_request(self, method, req_params=None, auth_params=None, file_params=None, retries=None): '''Make an HTTP request to a server method. The given method is called with any parameters set in ``req_params``. If auth is True, then the request is made with an authenticated session cookie. Note that path parameters should be set by adding onto the method, not via ``req_params``. :arg method: Method to call on the server. It's a url fragment that comes after the base_url set in __init__(). Note that any parameters set as extra path information should be listed here, not in ``req_params``. :kwarg req_params: dict containing extra parameters to send to the server :kwarg auth_params: dict containing one or more means of authenticating to the server. Valid entries in this dict are: :cookie: **Deprecated** Use ``session_id`` instead. If both ``cookie`` and ``session_id`` are set, only ``session_id`` will be used. A ``Cookie.SimpleCookie`` to send as a session cookie to the server :session_id: Session id to put in a cookie to construct an identity for the server :username: Username to send to the server :password: Password to use with username to send to the server :httpauth: If set to ``basic`` then use HTTP Basic Authentication to send the username and password to the server. This may be extended in the future to support other httpauth types than ``basic``. Note that cookie can be sent alone but if one of username or password is set the other must as well. Code can set all of these if it wants and all of them will be sent to the server. Be careful of sending cookies that do not match with the username in this case as the server can decide what to do in this case. :kwarg file_params: dict of files where the key is the name of the file field used in the remote method and the value is the local path of the file to be uploaded. If you want to pass multiple files to a single file field, pass the paths as a list of paths. :kwarg retries: if we get an unknown or possibly transient error from the server, retry this many times. Setting this to a negative number makes it try forever. Defaults to zero, no retries. number makes it try forever. Default to use the :attr:`retries` value set on the instance or in :meth:`__init__`. :returns: If ProxyClient is created with session_as_cookie=True (the default), a tuple of session cookie and data from the server. If ProxyClient was created with session_as_cookie=False, a tuple of session_id and data instead. :rtype: tuple of session information and data from server .. versionchanged:: 0.3.17 No longer send tg_format=json parameter. We rely solely on the Accept: application/json header now. .. versionchanged:: 0.3.21 * Return data as a Bunch instead of a DictContainer * Add file_params to allow uploading files ''' self.log.debug(b_('proxyclient.send_request: entered')) # Check whether we need to authenticate for this request session_id = None username = None password = None if auth_params: if 'session_id' in auth_params: session_id = auth_params['session_id'] elif 'cookie' in auth_params: warnings.warn(b_('Giving a cookie to send_request() to' ' authenticate is deprecated and will be removed in 0.4.' ' Please port your code to use session_id instead.'), DeprecationWarning, stacklevel=2) session_id = auth_params['cookie'].output(attrs=[], header='').strip() if 'username' in auth_params and 'password' in auth_params: username = auth_params['username'] password = auth_params['password'] elif 'username' in auth_params or 'password' in auth_params: raise AuthError(b_('username and password must both be set in' ' auth_params')) if not (session_id or username): raise AuthError(b_('No known authentication methods' ' specified: set "cookie" in auth_params or set both' ' username and password in auth_params')) # urljoin is slightly different than os.path.join(). Make sure method # will work with it. method = method.lstrip('/') # And join to make our url. url = urljoin(self.base_url, urllib.quote(method)) response = _PyCurlData() # The data we get back from the server data = None # decoded JSON via json.load() request = pycurl.Curl() # Fix encoding issue related to pycurl bug#1831680 request.setopt(pycurl.URL, to_bytes(url)) # Boilerplate so pycurl processes cookies request.setopt(pycurl.COOKIEFILE, '/dev/null') # Associate with the response to accumulate data request.setopt(pycurl.WRITEFUNCTION, response.write_data) # Follow redirect request.setopt(pycurl.FOLLOWLOCATION, True) request.setopt(pycurl.MAXREDIRS, 5) if self.insecure: # Don't check that the server certificate is valid # This flag should really only be set for debugging request.setopt(pycurl.SSL_VERIFYPEER, False) request.setopt(pycurl.SSL_VERIFYHOST, False) # Set standard headers headers = ['User-agent: %s' % self.useragent, 'Accept: application/json',] # This is a workaround # apache before 2.2.18 has a bug with Expect: continue headers # https://issues.apache.org/bugzilla/show_bug.cgi?id=47087 # curl triggers this by setting Expect: 100-continue headers # but putting data into the POST body that it expects the # server to process. Setting an empty Expect: header # overrides that. We should check whether this is still an # issue with apache 2.2.18 (or a patched apache) headers.append('Expect:') request.setopt(pycurl.HTTPHEADER, headers) # Files to upload if file_params: # pycurl wants files to be uploaded in a specific format. That # format is: # # [(file form field name, [(FORM_FILE, 'file local path'), ]), # (file form field name2, [(FORM_FILE, 'file2 local path'), ]) # ] file_data = [] for field_name, field_value in file_params.iteritems(): files = [] if isiterable(field_value): for filename in field_value: files.append((pycurl.FORM_FILE, filename)) else: files = [(pycurl.FORM_FILE, field_value)] file_data.append((field_name, files)) request.setopt(pycurl.HTTPPOST, file_data) # If we have a session_id, send it if session_id: # Anytime the session_id exists, send it so that visit tracking # works. Will also authenticate us if there's a need. request.setopt(pycurl.COOKIE, to_bytes('%s=%s;' % (self.session_name, session_id))) complete_params = req_params or {} if session_id: # Add the csrf protection token token = sha_constructor(session_id) complete_params.update({'_csrf_token': token.hexdigest()}) if username and password: if auth_params.get('httpauth', '').lower() == 'basic': # HTTP Basic auth login userpwd = '%s:%s' % (to_bytes(username), to_bytes(password)) request.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) request.setopt(pycurl.USERPWD, userpwd) else: # TG login # Adding this to the request data prevents it from being logged by # apache. complete_params.update({'user_name': to_bytes(username), 'password': to_bytes(password), 'login': '******'}) req_data = None if complete_params: # set None params to empty string otherwise the string 'None' gets # sent. for i in complete_params.iteritems(): if i[1] is None: complete_params[i[0]] = '' req_data = urllib.urlencode(complete_params, doseq=True) request.setopt(pycurl.POSTFIELDS, req_data) # If debug, give people our debug info self.log.debug(b_('Creating request %(url)s') % {'url': to_bytes(url)}) self.log.debug(b_('Headers: %(header)s') % {'header': to_bytes(headers, nonstring='simplerepr')}) if self.debug and req_data: debug_data = re.sub(r'(&?)password[^&]+(&?)', '\g<1>password=xxxxxxx\g<2>', req_data) self.log.debug(b_('Data: %(data)s') % {'data': to_bytes(debug_data)}) num_tries = 0 if retries is None: retries = self.retries while True: # run the request request.perform() # Check for auth failures # Note: old TG apps returned 403 Forbidden on authentication failures. # Updated apps return 401 Unauthorized # We need to accept both until all apps are updated to return 401. http_status = request.getinfo(pycurl.HTTP_CODE) if http_status in (401, 403): # Wrong username or password self.log.debug(b_('Authentication failed logging in')) raise AuthError(b_('Unable to log into server. Invalid' ' authentication tokens. Send new username and password')) elif http_status >= 400: if retries < 0 or num_tries < retries: # Retry the request num_tries += 1 self.log.debug(b_('Attempt #%(try)s failed') % {'try': num_tries}) time.sleep(0.5) continue # Fail and raise an error try: msg = httplib.responses[http_status] except (KeyError, AttributeError): msg = b_('Unknown HTTP Server Response') raise ServerError(url, http_status, msg) # Successfully returned data break # In case the server returned a new session cookie to us new_session = '' cookies = request.getinfo(pycurl.INFO_COOKIELIST) for cookie in cookies: host, isdomain, path, secure, expires, name, value = \ cookie.split('\t') if name == self.session_name: new_session = value break # Read the response json_string = response.data try: data = json.loads(json_string) except ValueError, e: # The response wasn't JSON data raise ServerError(url, http_status, b_('Error returned from' ' json module while processing %(url)s: %(err)s') % {'url': to_bytes(url), 'err': to_bytes(e)})