def fetch(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='', timeout=25): """Does a HTTP request with method `method` to `url`. Returns (status, headers, content) whereas `status` is an integer status code, `headers` is a dict containing the headers sent by the server and `content` is the body of the http response. Parameters to fetch are:: * `url` is the fully qualified request URL. It may contain query prameters. * `content` is the request body to be sent. It may be a dict which for all requests expect POST is converted to query parameters. If there are query parameters already in the `url` they are merged with `content`. For POST requests the data is encoded as application/x-www-form-urlencoded or multipart/form-data and encoded. If the parameter `multipart` is `True` or if one of the values in content has a `name` attribute (which is the case for file objects) multipart encoding is choosen. * `headers` is a dict of header values * `credentials` can be a user:password combination * `ua` should be an additional User Agent string * `timeout` is the maximum number of seconds the request might take. This is advisory and may not be enforced. """ myheaders = {'Accept-Encoding': 'gzip;q=1.0, *;q=0', 'User-Agent': '%s/huTools.http (gzip)' % ua} if headers: myheaders.update(headers) if method == 'POST': if hasattr(content, 'items'): # we assume content is a dict which needs to be encoded # decide to use multipart/form-data encoding or application/x-www-form-urlencoded for val in content.values(): if hasattr(val, 'read'): # file() or StringIO() multipart = True if multipart: datagen, mp_headers = poster_encode.multipart_encode(content) myheaders.update(mp_headers) content = "".join(datagen) else: myheaders.update({'Content-Type': 'application/x-www-form-urlencoded'}) content = tools.urlencode(content) else: # url parmater encoding if hasattr(content, 'items'): scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) qdict = cgi.parse_qs(query) # ugly Unicode issues, see http://bugs.python.org/issue1712522 qdict.update(content) query = tools.urlencode(qdict) url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) content = '' # convert all header values to strings (what about unicode?) for key, val in myheaders.items(): myheaders[key] = str(val) # add authentication if credentials and not 'Authorization' in myheaders.keys(): authheader = "Basic %s" % credentials.encode('base64').strip() myheaders["Authorization"] = authheader return request(url, method, content, myheaders, timeout)
def prepare_headers(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='', timeout=50, caching=None): """Prepares a request, returns (url, method, content, headers, timeout)""" myheaders = { 'Accept-Encoding': 'gzip', 'User-Agent': '%s/huTools.http (gzip)' % ua } if headers: myheaders.update(headers) if method in ['POST', 'PUT']: if hasattr(content, 'items'): # we assume content is a dict which needs to be encoded # decide to use multipart/form-data encoding or application/x-www-form-urlencoded for val in content.values(): if hasattr(val, 'read'): # file() or StringIO() multipart = True break if multipart: datagen, mp_headers = poster_encode.multipart_encode(content) myheaders.update(mp_headers) content = "".join(datagen) else: myheaders.update( {'Content-Type': 'application/x-www-form-urlencoded'}) content = urlencode(content) else: # url parmater encoding if hasattr(content, 'items'): scheme, netloc, path, params, query, fragment = urlparse.urlparse( url) qdict = urlparse.parse_qs(query) # ugly Unicode issues, see http://bugs.python.org/issue1712522 qdict.update(content) query = urlencode(qdict) url = urlparse.urlunparse( (scheme, netloc, path, params, query, fragment)) content = '' # convert all header values to strings (what about unicode?) for key, val in myheaders.items(): myheaders[key] = str(val) # add authentication if credentials and not 'Authorization' in myheaders.keys(): myheaders["Authorization"] = 'Basic %s' % base64.b64encode(credentials) return url, method, content, myheaders, timeout, caching
def prepare_headers(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='', timeout=50, caching=None): """Prepares a request, returns (url, method, content, headers, timeout)""" myheaders = {'Accept-Encoding': 'gzip', 'User-Agent': '%s/huTools.http (gzip)' % ua} if headers: myheaders.update(headers) if method == 'POST': if hasattr(content, 'items'): # we assume content is a dict which needs to be encoded # decide to use multipart/form-data encoding or application/x-www-form-urlencoded for val in content.values(): if hasattr(val, 'read'): # file() or StringIO() multipart = True if multipart: datagen, mp_headers = poster_encode.multipart_encode(content) myheaders.update(mp_headers) content = "".join(datagen) else: myheaders.update({'Content-Type': 'application/x-www-form-urlencoded'}) content = urlencode(content) else: # url parmater encoding if hasattr(content, 'items'): scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) qdict = cgi.parse_qs(query) # ugly Unicode issues, see http://bugs.python.org/issue1712522 qdict.update(content) query = urlencode(qdict) url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) content = '' # convert all header values to strings (what about unicode?) for key, val in myheaders.items(): myheaders[key] = str(val) # add authentication if credentials and not 'Authorization' in myheaders.keys(): # ''.encode('b64') zerbricht lange Strings (also bei langen Credentials) # in mehrere Zeilen, was einer Verwendung als Wert im HTTP-Header extrem # abtraeglich ist. Aufgetaucht ist das Problem bei den Login-Daten fuer # den Import der Shopify-Auftraege aus dem hySkate-Shop. Zur Loesung werden # deshalb alle moeglichen Einzelzeilen wieder zu einer Zeile zusammengezogen. authheader = 'Basic %s' % ''.join(credentials.encode('base64').strip().split()) myheaders["Authorization"] = authheader return url, method, content, myheaders, timeout, caching
def fetch(url, content='', method='GET', credentials=None, headers=None, multipart=False, ua='', timeout=25): """Does a HTTP request with method `method` to `url`. Returns (status, headers, content) whereas `status` is an integer status code, `headers` is a dict containing the headers sent by the server and `content` is the body of the http response. Parameters to fetch are:: * `url` is the fully qualified request URL. It may contain query prameters. * `content` is the request body to be sent. It may be a dict which for all requests expect POST is converted to query parameters. If there are query parameters already in the `url` they are merged with `content`. For POST requests the data is encoded as application/x-www-form-urlencoded or multipart/form-data and encoded. If the parameter `multipart` is `True` or if one of the values in content has a `name` attribute (which is the case for file objects) multipart encoding is choosen. * `headers` is a dict of header values * `credentials` can be a user:password combination * `ua` should be an additional User Agent string * `timeout` is the maximum number of seconds the request might take. This is advisory and may not be enforced. """ myheaders = { # Für [LH#1003] als Hotfix erstmal abgeschaltet # urlfetch.fetch erkennt den Response Header "content-encoding: gzip" scheinbar nicht #'Accept-Encoding': 'gzip;q=1.0, *;q=0', 'User-Agent': '%s/huTools.http (gzip)' % ua } if headers: myheaders.update(headers) if method == 'POST': if hasattr(content, 'items'): # we assume content is a dict which needs to be encoded # decide to use multipart/form-data encoding or application/x-www-form-urlencoded for val in content.values(): if hasattr(val, 'read'): # file() or StringIO() multipart = True if multipart: datagen, mp_headers = poster_encode.multipart_encode(content) myheaders.update(mp_headers) content = "".join(datagen) else: myheaders.update( {'Content-Type': 'application/x-www-form-urlencoded'}) content = tools.urlencode(content) else: # url parmater encoding if hasattr(content, 'items'): scheme, netloc, path, params, query, fragment = urlparse.urlparse( url) qdict = cgi.parse_qs(query) # ugly Unicode issues, see http://bugs.python.org/issue1712522 qdict.update(content) query = tools.urlencode(qdict) url = urlparse.urlunparse( (scheme, netloc, path, params, query, fragment)) content = '' # convert all header values to strings (what about unicode?) for key, val in myheaders.items(): myheaders[key] = str(val) # add authentication if credentials and not 'Authorization' in myheaders.keys(): authheader = "Basic %s" % credentials.encode('base64').strip() myheaders["Authorization"] = authheader return request(url, method, content, myheaders, timeout)