def _find_headers(self):
# Cached?
cache_file = self._headers_file_name()
try:
headers = httplib.HTTPMessage(open(cache_file))
except:
headers = None
# Retrieve
if not headers:
url = self.url()
host = str(urlparse.urlparse(url)[1])
def block():
try:
connection = urllib2.urlopen(url)
headers = connection.info()
connection.close()
except urllib2.HTTPError:
return 'URL could not be fetched', None
return None, headers
headers = logger.with_action_log('Connecting to %s' % (host, ),
block)
# Save
if headers:
try:
open(cache_file, 'w').write(str(headers))
except:
pass
else:
headers = httplib.HTTPMessage(StringIO.StringIO(''), seekable=0)
return headers
def __init__(self, content):
if PY2x:
fp = StringIO(content)
self._info = httplib.HTTPMessage(fp)
else:
self._info = httplib.HTTPMessage()
# Adjust to testdata.
l = content.split(':')
if len(l) > 1:
# Get the type by just
# using the data at the end.
t = l[-1].strip()
self._info.set_type(t)
def _parse_headers(self, status_and_headers):
"""Transform the headers provided by curl into an HTTPMessage"""
status_and_headers.seek(0)
# Ignore status line
status_and_headers.readline()
msg = httplib.HTTPMessage(status_and_headers)
return msg
def _make_response(self, result, url):
data = "\r\n".join(["%s: %s" % (k, v) for k, v in result.header_items])
headers = httplib.HTTPMessage(StringIO(data))
response = urllib.addinfourl(StringIO(result.data), headers, url)
code, msg = result.status.split(None, 1)
response.code, response.msg = int(code), msg
return response
def exec_open (self, req):
path = req.get_selector()
args = path.split("?", 1)
if len(args) == 1: args.append('')
#print "args ", args
# Prepare CGI-like environment
os.putenv ('GATEWAY_INTERFACE', 'CGI/1.1')
os.putenv ('HTTP_ACCEPT_ENCODING', req.headers.get ('Accept-encoding'))
os.putenv ('HTTP_USER_AGENT', 'DBS-CGI-Direct-call')
os.putenv ('REQUEST_METHOD', 'POST')
os.putenv ('CONTENT_LENGTH', str(req.headers.get ('Content-length')))
os.putenv ('CONTENT_TYPE', req.headers.get ('Content-type'))
os.putenv ('QUERY_STRING', args[1])
os.putenv ('REQUEST_URI', path)
os.putenv ('SCRIPT_NAME', args[0])
os.putenv ('SERVER_NAME', 'localhost')
os.putenv ('SERVER_PORT', str(80))
os.putenv ('SERVER_PROTOCOL', 'HTTP/1.1')
os.putenv ('SERVER_SOFTWARE', 'Builtin')
# Open subprocess and write form data
r, w = os.popen2(args[0])
r.write (req.get_data())
r.close ()
# Read back headers, then leave the body to be read
msg = httplib.HTTPMessage (w, 0)
msg.fp = None
return urllib.addinfourl (w, msg, path)
def parse_response(meta):
""" Parse Cache Metadata, returns (method, status, header) """
try:
method = meta['request-method']
except KeyError:
method = "UNKNOWN"
try:
header = meta['response-head'].splitlines(True)
status = header[0].split(None, 2)[1]
header = httplib.HTTPMessage(StringIO.StringIO("".join(header[1:])))
except KeyError:
status = "UNKNOWN"
header = httplib.HTTPMessage(StringIO.StringIO())
return (method, status, header)
def gitweb(request):
cigenv = {
'REQUEST_METHOD': request.method,
'QUERY_STRING': request.META['QUERY_STRING'],
'GITWEB_SITENAME': 'gitub',
'GITWEB_CONFIG': getattr(settings, 'PROJECT_PATH') + '/gitweb.conf',
}
os.environ.update(cigenv)
p = subprocess.Popen([getattr(settings, 'PROJECT_PATH') + '/gitweb.cgi'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
res = httplib.HTTPResponse(FakeSock(), method='GET')
res.fp = StringIO(stdout)
res.begin()
if not res.getheaders():
msg = httplib.HTTPMessage(StringIO(stdout))
else:
msg = res.msg
#remove headers
while True:
line = res.fp.readline()
if line == '\r\n': break
response = HttpResponse(res.read(), mimetype=msg.getheader('content-type'))
for item in msg.items():
response[item[0]] = item[1]
return response
def decode(page):
"gunzip or deflate a compressed page"
#print page.info().headers
encoding = page.info().get("Content-Encoding")
if encoding in ('gzip', 'x-gzip', 'deflate'):
from cStringIO import StringIO
# cannot seek in socket descriptors, so must get content now
content = page.read()
if encoding == 'deflate':
import zlib
fp = StringIO(zlib.decompress(content))
else:
import gzip
fp = gzip.GzipFile('', 'rb', 9, StringIO(content))
# remove content-encoding header
headers = httplib.HTTPMessage(StringIO(""))
ceheader = re.compile(r"(?i)content-encoding:")
for h in page.info().keys():
if not ceheader.match(h):
headers[h] = page.info()[h]
newpage = urllib.addinfourl(fp, headers, page.geturl())
# Propagate code, msg through
if hasattr(page, 'code'):
newpage.code = page.code
if hasattr(page, 'msg'):
newpage.msg = page.msg
return newpage
return page
def OnHeaderEnd(self):
self.headfp.seek(0)
self.msg = httplib.HTTPMessage(self.headfp)
# are we using the chunked-style of transfer encoding?
tr_enc = self.msg.getheader('transfer-encoding')
if tr_enc and tr_enc.lower() == "chunked":
self.chunked = 1
self.chunk_left = None
else:
self.chunked = 0
# do we have a Content-Length?
# NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
length = self.msg.getheader('content-length')
if length and not self.chunked:
try:
self.length = int(length)
except ValueError:
self.length = None
else:
self.length = None
# does the body have a fixed length? (of zero)
if (self.status == httplib.NO_CONTENT
or self.status == httplib.NOT_MODIFIED
or 100 <= self.status < 200):
self.length = 0
if self.status == httplib.OK:
self.OnGetHeaderSucc()
else:
self.OnGetHeaderFailed()
def serve_php(self, environ, start_response, php_args, php_env, content):
try:
p = subprocess.Popen(php_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=php_env, cwd=self.cwd)
except Exception as e:
start_response('500 Internal Server Error', [('Content-Type', 'text/html')])
return [traceback.format_exc()]
stdout, stderr = p.communicate(content)
message = httplib.HTTPMessage(cStringIO.StringIO(stdout))
assert 'Content-Type' in message, 'invalid CGI response: %r' % stdout
if 'Status' in message:
status = message['Status']
del message['Status']
else:
status = '200 OK'
# Ensures that we avoid merging repeat headers into a single header,
# allowing use of multiple Set-Cookie headers.
headers = []
for name in message:
for value in message.getheaders(name):
headers.append((name, value))
start_response(status, headers)
return [message.fp.read()]
def gae_urlfetch(method, url, headers, payload, app_server, **kwargs):
if payload:
if len(payload
) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers:
zpayload = zlib.compress(payload)[2:-4]
if len(zpayload) < len(payload):
payload = zpayload
headers['Content-Encoding'] = 'deflate'
headers['Content-Length'] = str(len(payload))
# GAE donot allow set `Host` header
if 'Host' in headers:
del headers['Host']
metadata = 'G-Method:%s\nG-Url:%s\n%s' % (method, url, ''.join(
'G-%s:%s\n' % (k, v) for k, v in kwargs.items() if v))
skip_headers = https_manager.skip_headers
metadata += ''.join('%s:%s\n' % (k.title(), v) for k, v in headers.items()
if k not in skip_headers)
# prepare GAE request
request_method = 'POST'
request_headers = {}
metadata = zlib.compress(metadata)[2:-4]
payload = '%s%s%s' % (struct.pack('!h', len(metadata)), metadata, payload)
request_headers['Content-Length'] = str(len(payload))
# post data
#start_time = time.time()
response = https_manager.request(request_method, app_server, payload,
request_headers)
#stop_time = time.time()
#cost_time = stop_time - start_time
if hasattr(response, "status"):
response.app_status = response.status
else:
return response
#logging.debug("request time:%d status:%d url:%s ", int(cost_time * 1000), response.status, url)
response.app_options = response.getheader('X-GOA-Options', '')
if response.status != 200:
return response
data = response.read(4)
if len(data) < 4:
response.status = 502
response.fp = io.BytesIO(
b'connection aborted. too short leadtype data=' + data)
response.read = response.fp.read
return response
response.status, headers_length = struct.unpack('!hh', data)
data = response.read(headers_length)
if len(data) < headers_length:
response.status = 502
response.fp = io.BytesIO(
b'connection aborted. too short headers data=' + data)
response.read = response.fp.read
return response
response.msg = httplib.HTTPMessage(
io.BytesIO(zlib.decompress(data, -zlib.MAX_WBITS)))
return response
def my_urlopen(obj, timeout=None):
uri = get_uri(obj)
self.assertEquals(uri, f.uri)
return urllib.addinfourl(
StringIO(test_data),
httplib.HTTPMessage(
StringIO('Content-Length: {}'.format(len(test_data) + 1))),
f.uri)
def my_urlopen(obj, timeout=None):
uri = get_uri(obj)
seen_uris.append(uri)
return urllib.addinfourl(
StringIO(test_data),
httplib.HTTPMessage(
StringIO('Content-Length: {}\r\nDigest: SHA-256={}'.format(
len(test_data), digest))), f.uri)
def my_urlopen(obj, timeout=None):
uri = get_uri(obj)
self.assertEqual(uri, fileset_uri)
fp = StringIO('\n'.join(files + ('', )))
msg = httplib.HTTPMessage(fp=StringIO(
'Content-Length: {}'.format(len(fp.getvalue()) + 1)),
seekable=True)
return urllib.addinfourl(fp, msg, uri)
def to_addinfourl(response):
"""Convert a `django.http.HttpResponse` to a `urllib2.addinfourl`."""
headers_raw = response.serialize_headers()
headers = httplib.HTTPMessage(io.BytesIO(headers_raw))
return urllib2.addinfourl(fp=io.BytesIO(response.content),
headers=headers,
url=None,
code=response.status_code)
def get_message(in_):
header = find_until_newline(in_)
message = mimetools.Message(StringIO.StringIO(header))
content_length = message.getheader("content-length")
body = ""
if (content_length != None):
content_length = int(content_length)
body = get_body(in_, content_length)
return httplib.HTTPMessage(StringIO.StringIO(header + body)), body
def readCacheMeta(fname):
if not fileExists(fname): return None
try:
fid = file(fname, 'rb')
info = httplib.HTTPMessage(fid)
fid.close()
return info
except:
return None
def __enter__(self):
self.request_context = mock.patch.object(httplib.HTTPConnection,
'request')
self.request_context.return_value = None
self.request_mock = self.request_context.__enter__()
self.fixture_file = open(
join(dirname(__file__), 'fixtures', self.fixture), 'rb')
# Read through the request.
preamble_line = self.fixture_file.readline().strip()
try:
self.method, self.uri, http_version = preamble_line.split(None, 2)
except ValueError:
raise ValueError(
"Couldn't parse preamble line from fixture file %r; does it have a fixture in it?"
% self.fixture)
msg = httplib.HTTPMessage(self.fixture_file, 0)
self.headers = dict(
(k, v.strip())
for k, v in (header.split(':', 1) for header in msg.headers))
msg.fp = None
# Read through to the vertical space.
def nextline(fp):
while True:
try:
line = fp.readline()
except EOFError:
return
if not line or line.startswith('\x16'):
return
yield line
body = ''.join(nextline(
self.fixture_file)) # exhaust the request either way
self.body = None
if self.method in ('PUT', 'POST'):
if 'Content-Type' in self.headers:
if 'application/xml' in self.headers['Content-Type']:
self.body = xml(body)
else:
self.body = body
# Set up the response returner.
sock = mock.Mock()
sock.makefile = mock.Mock(return_value=self.fixture_file)
response = httplib.HTTPResponse(sock, method=self.method)
response.begin()
self.response_context = mock.patch.object(httplib.HTTPConnection,
'getresponse',
lambda self: response)
self.response_mock = self.response_context.__enter__()
return self
def my_urlopen(obj, timeout=None):
uri = get_uri(obj)
self.assertEqual(uri, fileset_uri)
fp = StringIO('\n'.join(files + ('', )))
digest = base64.b64encode(hashlib.sha256(fp.getvalue()).digest())
msg = httplib.HTTPMessage(fp=StringIO(
'Content-Length: {}\r\nDigest: SHA-256={}'.format(
len(fp.getvalue()), digest)),
seekable=True)
return urllib.addinfourl(fp, msg, uri)
def __init__(self, content, status_code, content_was_truncated, final_url, headers):
import httplib
from StringIO import StringIO
self.content = content
self.status_code = status_code
self.content_was_truncated = content_was_truncated
self.final_url = final_url
self.header_msg = httplib.HTTPMessage(StringIO(''.join(
["%s: %s\n" % (k, v) for k, v in headers.items()] + ["\n"])))
self.headers = self.header_msg.items()
def do_rsync(self, source, attrs=[]):
options = dict()
for attr in attrs:
k, _, v = attr.partition('=')
options[k] = v
options.setdefault('rsync_path', self.rsync_path)
options.setdefault('rsync_rsh', self.rsync_rsh)
cmd_args = [options['rsync_path'], '-t', '--whole-file']
if options['rsync_rsh']:
cmd_args.extend(('-e', options['rsync_rsh']))
fn = source.rpartition('/')[2]
tf = tempfile.mktemp(prefix='rsync--{}.'.format(fn), dir=self.tmpdir)
cmd_args.extend((source, tf))
logger.debug('Callling {}'.format(' '.join(map(pipes.quote,
cmd_args))))
stderr = tempfile.TemporaryFile(dir=self.tmpdir)
try:
subprocess.check_call(cmd_args, stderr=stderr)
tf_stat = os.stat(tf)
fp = open(tf)
except subprocess.CalledProcessError:
stderr.seek(0)
raise urllib2.URLError('rsync error: {}'.format(stderr.read()))
finally:
try:
os.unlink(tf)
except OSError as e:
logger.error('Error unlinking {}: {}'.format(tf, e))
headers = StringIO()
mtype = mimetypes.guess_type(source)[0]
if mtype:
print('Content-type: {}'.format(mtype), file=headers)
logger.debug('Content-type: {}'.format(mtype))
print('Content-length: {:0d}'.format(tf_stat.st_size), file=headers)
logger.debug('Content-length: {:0d}'.format(tf_stat.st_size))
print('Last-modified: {}'.format(
email.utils.formatdate(tf_stat.st_mtime, usegmt=True)),
file=headers)
logger.debug('Last-modified: {}'.format(
email.utils.formatdate(tf_stat.st_mtime, usegmt=True)))
headers.seek(0)
msg = httplib.HTTPMessage(fp=headers, seekable=True)
return urllib.addinfourl(fp, msg, source)
def __init__(self, cacheLocation,url,setCacheHeader=True):
self.cacheLocation = cacheLocation
hash = hashlib.md5(url).hexdigest()
StringIO.StringIO.__init__(self, file(self.cacheLocation + "/" + hash+".body").read())
self.url = url
self.code = 200
self.msg = "OK"
headerbuf = file(self.cacheLocation + "/" + hash+".headers").read()
if setCacheHeader:
headerbuf += "x-cache: %s/%s\r\n" % (self.cacheLocation,hash)
self.headers = httplib.HTTPMessage(StringIO.StringIO(headerbuf))
def my_urlopen(obj, timeout=None):
uri = get_uri(obj)
msg = httplib.HTTPMessage(StringIO())
if uri == fileset_uri:
return urllib.addinfourl(StringIO('\n'.join(fileset + ('', ))),
msg, uri)
else:
self.assertTrue(uri.startswith('{}/'.format(uri_base)))
return urllib.addinfourl(
StringIO('{}'.format(uri[1 + len(uri_base):])), msg, uri)
def __init__(self, cache_location, url, set_cache_header=True):
self.cache_location = cache_location
hpath, bpath = calculate_cache_path(cache_location, url)
StringIO.StringIO.__init__(self, file(bpath, "rb").read())
self.url = url
self.code = 200
self.msg = "OK"
headerbuf = file(hpath, "rb").read()
if set_cache_header:
headerbuf += "x-local-cache: %s\r\n" % (bpath)
self.headers = httplib.HTTPMessage(StringIO.StringIO(headerbuf))
def make_response(status_code, content, content_type=None):
"""Return a similar response to that which `urllib2` returns."""
if content_type is None:
headers_raw = b""
else:
if isinstance(content_type, unicode):
content_type = content_type.encode("ascii")
headers_raw = b"Content-Type: %s" % content_type
headers = httplib.HTTPMessage(io.BytesIO(headers_raw))
return urllib2.addinfourl(fp=io.BytesIO(content),
headers=headers,
url=None,
code=status_code)
def get_info_from_header(buffered_header):
response = httplib.HTTPMessage(StringIO(buffered_header), 0)
# On OSX you seem to need to read the headers, while on Windows you do not. The two
# seem to be mutually exclusive - one will not work on the other. This is a workaround
# to check whether we need to explicitly read the headers or not. If both attempts fail,
# we manually parse the headers.
# Note: We use content-type to determine whether the headers have been parsed out
# correctly by the HTTPMessage, since it is common to find that in the headers.
# It is not required, however, so it could be missing even if the headers were
# parsed correctly.
# Attempt 1: Not explicitly reading the headers
content_type = response.getheader("content-type")
content_length = response.getheader("content-length")
transfer_encoding = response.getheader("transfer-encoding")
# Attempt 2: Maybe we needed to explicitly read the headers
if not content_type:
response.readheaders()
content_type = response.getheader("content-type")
content_length = response.getheader("content-length")
transfer_encoding = response.getheader("transfer-encoding")
# Attempt 3: Parse the fields from the header manually
if not content_type:
header_lower = buffered_header.lower()
if "content-length" in header_lower:
content_length_begin = header_lower.find("content-length")
content_length_end = header_lower[content_length_begin:].find("\n") + content_length_begin
content_length_header = header_lower[content_length_begin:content_length_end + 1]
if VERBOSE: logging.debug("Manually parsed out content length header: %s" % content_length_header)
if content_length_header:
content_length = content_length_header.split(":")[1].strip()
if VERBOSE: logging.debug("Parsed out this content length: %s" % content_length)
else:
content_length = None
if "transfer-encoding" in header_lower:
transfer_encoding_begin = header_lower.find("transfer-encoding")
transfer_encoding_end = header_lower[transfer_encoding_begin:].find("\n") + transfer_encoding_begin
transfer_encoding_header = header_lower[transfer_encoding_begin:transfer_encoding_end + 1]
if VERBOSE: logging.debug("Manually parsed out transfer_encoding header: %s" % transfer_encoding_header)
if transfer_encoding_header:
transfer_encoding = transfer_encoding_header.split(":")[1].strip()
if VERBOSE: logging.debug("Parsed out this transfer_encoding: %s" % transfer_encoding)
else:
transfer_encoding = None
return (content_length, transfer_encoding)
def read_range_definition(self):
"""Read a new range definition in a multi parts message.
Parse the headers including the empty line following them so that we
are ready to read the data itself.
"""
self._headers = httplib.HTTPMessage(self._file, seekable=0)
# Extract the range definition
content_range = self._headers.getheader('content-range', None)
if content_range is None:
raise errors.InvalidHttpResponse(
self._path,
'Content-Range header missing in a multi-part response')
self.set_range_from_header(content_range)
def from_headers (strheader):
"""Parse cookie data from a string in HTTP header (RFC 2616) format.
@return: tuple (headers, scheme, host, path)
@raises: ValueError for incomplete or invalid data
"""
fp = StringIO(strheader)
headers = httplib.HTTPMessage(fp, seekable=True)
if "Host" not in headers:
raise ValueError("Required header 'Host:' missing")
host = headers["Host"]
scheme = headers.get("Scheme", "http")
path= headers.get("Path", "/")
return (headers, scheme, host, path)
def testscheme_open(self, req):
try:
selector = req.get_selector()
path = 'tests/data/' + selector.split('/')[-1] + '.json'
if pkg_resources.resource_exists(PACKAGE_NAME, path):
return urllib.addinfourl(
pkg_resources.resource_stream(PACKAGE_NAME, path),
httplib.HTTPMessage(open('/dev/null')), req.get_full_url(),
200)
else:
raise urllib2.URLError('Not found')
except Exception:
raise urllib2.URLError('Not found')
def __init__(self, response_proto):
"""Constructor.
Args:
response_proto: the URLFetchResponse proto buffer to wrap.
"""
self.__pb = response_proto
self.content = response_proto.content()
self.status_code = response_proto.statuscode()
self.content_was_truncated = response_proto.contentwastruncated()
self.final_url = response_proto.finalurl() or None
self.header_msg = httplib.HTTPMessage(
StringIO.StringIO(''.join(['%s: %s\n' % (h.key(), h.value())
for h in response_proto.header_list()] + ['\n'])))
self.headers = _CaselessDict(self.header_msg.items())