def fetch(url,
postdata=None,
server=None,
port=None,
protocol=None,
ok_codes=None,
**kw):
""" Replace the fetch with something a bit more direct """
t_protocol, t_server, t_url, x, t_args, x = urlparse.urlparse(url)
t_port = None
url = t_url[1:]
if server is None:
if ':' in t_server:
t_server, t_port = t_server.split(':')
server = t_server
if port is None and t_port is not None:
port = t_port
if protocol is None:
protocol = t_protocol
#do socket call here.
sock = Sock(server, port)
response = sock.exchange("GET %s\n" % url)
sock.close()
error_content = []
code, user = response.strip().split(' ', 1)
response = HTTPResponse([], protocol, server, port, url, int(code), user,
{}, user, error_content)
return response
def WF_fetch(self,
url,
postdata=None,
server=None,
port=None,
protocol=None,
ok_codes=None,
key_file=None,
cert_file=None,
method="GET",
consumer=None):
'''Run a single test request to the indicated url. Use the POST data
if supplied. Accepts key and certificate file paths for https (ssl/tls)
connections.
Raises failureException if the returned data contains any of the
strings indicated to be Error Content.
Returns a HTTPReponse object wrapping the response from the server.
'''
# see if the url is fully-qualified (not just a path)
t_protocol, t_server, t_url, x, t_args, x = urlparse.urlparse(url)
if t_server:
protocol = t_protocol
if ':' in t_server:
server, port = t_server.split(':')
else:
server = t_server
if protocol == 'http':
port = '80'
else:
port = '443'
url = t_url
if t_args:
url = url + '?' + t_args
# ignore the machine name if the URL is for localhost
if t_server == 'localhost':
server = None
elif not server:
# no server was specified with this fetch, or in the URL, so
# see if there's a base URL to use.
base = self.get_base_url()
if base:
t_protocol, t_server, t_url, x, x, x = urlparse.urlparse(base)
if t_protocol:
protocol = t_protocol
if t_server:
server = t_server
if t_url:
url = urlparse.urljoin(t_url, url)
# TODO: allow override of the server and port from the URL!
if server is None:
server = self.server
if port is None:
port = self.port
if protocol is None:
protocol = self.protocol
if ok_codes is None:
ok_codes = self.expect_codes
webproxy = {}
if protocol == 'http':
try:
proxystring = os.environ["http_proxy"].replace("http://", "")
webproxy['host'] = proxystring.split(":")[0]
webproxy['port'] = int(proxystring.split(":")[1])
except (KeyError, IndexError, ValueError):
webproxy = False
if webproxy:
h = httplib.HTTPConnection(webproxy['host'], webproxy['port'])
else:
h = httplib.HTTP(server, int(port))
if int(port) == 80:
host_header = server
else:
host_header = '%s:%s' % (server, port)
elif protocol == 'https':
#if httpslib is None:
#raise ValueError, "Can't fetch HTTPS: M2Crypto not installed"
# FL Patch -------------------------
try:
proxystring = os.environ["https_proxy"].replace("http://",
"").replace(
"https://", "")
webproxy['host'] = proxystring.split(":")[0]
webproxy['port'] = int(proxystring.split(":")[1])
except (KeyError, IndexError, ValueError):
webproxy = False
# patched to use the given key and cert file
if webproxy:
h = httplib.HTTPSConnection(webproxy['host'], webproxy['port'],
key_file, cert_file)
else:
h = httplib.HTTPS(server, int(port), key_file, cert_file)
# FL Patch end -------------------------
if int(port) == 443:
host_header = server
else:
host_header = '%s:%s' % (server, port)
else:
raise ValueError(protocol)
headers = []
params = None
if postdata is not None:
if webproxy:
h.putrequest(method.upper(),
"%s://%s%s" % (protocol, host_header, url))
else:
# Normal post
h.putrequest(method.upper(), url)
if postdata:
if isinstance(postdata, Data):
# User data and content_type
params = postdata.data
if postdata.content_type:
headers.append(('Content-type', postdata.content_type))
else:
# Check for File upload
is_multipart = False
for field, value in postdata:
if isinstance(value, Upload):
# Post with a data file requires multipart mimeencode
is_multipart = True
break
if is_multipart:
params = mimeEncode(postdata)
headers.append(
('Content-type',
'multipart/form-data; boundary=%s' % BOUNDARY))
else:
params = urlencode(postdata)
headers.append(
('Content-type', 'application/x-www-form-urlencoded'))
headers.append(('Content-length', str(len(params))))
else:
if webproxy:
h.putrequest(method.upper(),
"%s://%s%s" % (protocol, host_header, url))
else:
h.putrequest(method.upper(), url)
# Other Full Request headers
if self.authinfo:
headers.append(('Authorization', "Basic %s" % self.authinfo))
if not webproxy:
# HTTPConnection seems to add a host header itself.
# So we only need to do this if we are not using a proxy.
headers.append(('Host', host_header))
# FL Patch -------------------------
for key, value in self.extra_headers:
headers.append((key, value))
# FL Patch end ---------------------
# Send cookies
# - check the domain, max-age (seconds), path and secure
# (http://www.ietf.org/rfc/rfc2109.txt)
cookies_used = []
cookie_list = []
for domain, cookies in self.cookies.items():
# check cookie domain
if not server.endswith(domain) and domain[1:] != server:
continue
for path, cookies in cookies.items():
# check that the path matches
urlpath = urlparse.urlparse(url)[2]
if not urlpath.startswith(path) and not (path == '/'
and urlpath == ''):
continue
for sendcookie in cookies.values():
# and that the cookie is or isn't secure
if sendcookie['secure'] and protocol != 'https':
continue
# TODO: check for expires (max-age is working)
# hard coded value that application can use to work
# around expires
if sendcookie.coded_value in ('"deleted"', "null", "deleted"):
continue
cookie_list.append("%s=%s;" %
(sendcookie.key, sendcookie.coded_value))
cookies_used.append(sendcookie.key)
if cookie_list:
headers.append(('Cookie', ' '.join(cookie_list)))
# check that we sent the cookies we expected to
if self.expect_cookies is not None:
assert cookies_used == self.expect_cookies, \
"Didn't use all cookies (%s expected, %s used)"%(
self.expect_cookies, cookies_used)
# write and finish the headers
for header in headers:
h.putheader(*header)
h.endheaders()
if self.debug_headers:
for header in headers:
print("Putting header -- %s: %s" % header)
if params is not None:
h.send(params)
# handle the reply
if webproxy:
r = h.getresponse()
errcode = r.status
errmsg = r.reason
headers = r.msg
if headers is None or 'content-length' in headers and headers[
'content-length'] == "0":
data = None
else:
data = r.read()
response = HTTPResponse(self.cookies, protocol, server, port, url,
errcode, errmsg, headers, data,
self.error_content)
else:
# get the body and save it
errcode, errmsg, headers = h.getreply()
if headers is None or 'content-length' in headers and headers[
'content-length'] == "0":
response = HTTPResponse(self.cookies, protocol, server, port, url,
errcode, errmsg, headers, None,
self.error_content)
else:
f = h.getfile()
g = cStringIO.StringIO()
if consumer is None:
d = f.read()
else:
d = f.readline(1)
while d:
g.write(d)
if consumer is None:
d = f.read()
else:
ret = consumer(d)
if ret == 0:
# consumer close connection
d = None
else:
d = f.readline(1)
response = HTTPResponse(self.cookies, protocol, server,
port, url, errcode, errmsg, headers,
g.getvalue(), self.error_content)
f.close()
if errcode not in ok_codes:
if VERBOSE:
sys.stdout.write('e')
sys.stdout.flush()
raise HTTPError(response)
# decode the cookies
if self.accept_cookies:
try:
# decode the cookies and update the cookies store
decodeCookies(url, server, headers, self.cookies)
except:
if VERBOSE:
sys.stdout.write('c')
sys.stdout.flush()
raise
# Check errors
if self.error_content:
data = response.body
for content in self.error_content:
if data.find(content) != -1:
msg = "Matched error: %s" % content
if hasattr(self, 'results') and self.results:
self.writeError(url, msg)
self.log('Matched error' + repr((url, content)), data)
if VERBOSE:
sys.stdout.write('c')
sys.stdout.flush()
raise self.failureException(msg)
if VERBOSE:
sys.stdout.write('_')
sys.stdout.flush()
return response