def get_file(self, url, output_stream, max_size=None, headers=None):
"""GETs a file from a given URL
Args:
url (str): The URL to GET
output_stream (file): File to write the response body to.
headers (dict[str, List[str]]|None): If not None, a map from
header name to a list of values for that header
Returns:
A (int,dict,string,int) tuple of the file length, dict of the response
headers, absolute URI of the response and HTTP response code.
"""
actual_headers = {
b"User-Agent": [self.user_agent],
}
if headers:
actual_headers.update(headers)
response = yield self.request(
"GET",
url.encode("ascii"),
headers=Headers(actual_headers),
)
resp_headers = dict(response.headers.getAllRawHeaders())
if 'Content-Length' in resp_headers and resp_headers[
'Content-Length'] > max_size:
logger.warn("Requested URL is too large > %r bytes" %
(self.max_size, ))
raise SynapseError(
502,
"Requested file is too large > %r bytes" % (self.max_size, ),
Codes.TOO_LARGE,
)
if response.code > 299:
logger.warn("Got %d when downloading %s" % (response.code, url))
raise SynapseError(
502,
"Got error %d" % (response.code, ),
Codes.UNKNOWN,
)
# TODO: if our Content-Type is HTML or something, just read the first
# N bytes into RAM rather than saving it all to disk only to read it
# straight back in again
try:
length = yield make_deferred_yieldable(
_readBodyToFile(
response,
output_stream,
max_size,
))
except Exception as e:
logger.exception("Failed to download body")
raise SynapseError(
502,
("Failed to download remote body: %s" % e),
Codes.UNKNOWN,
)
defer.returnValue((length, resp_headers, response.request.absoluteURI,
response.code), )
def test_rawHeadersTypeChecking(self):
"""
L{Headers.setRawHeaders} requires values to be of type sequence
"""
h = Headers()
self.assertRaises(TypeError, h.setRawHeaders, "key", {"Foo": "bar"})
def test_hasHeaderFalse(self):
"""
L{Headers.hasHeader} returns C{False} when the given header is not
found.
"""
self.assertFalse(Headers().hasHeader("test\u00E1"))
def store(self, key, value):
self.cacheStorage.put(key, Headers(), value)
def request(self, method, url, **kwargs):
"""
See :func:`treq.request()`.
"""
method = method.encode('ascii').upper()
if isinstance(url, unicode):
parsed_url = URL.from_text(url)
else:
parsed_url = URL.from_text(url.decode('ascii'))
# Join parameters provided in the URL
# and the ones passed as argument.
params = kwargs.get('params')
if params:
parsed_url = parsed_url.replace(
query=parsed_url.query + tuple(_coerced_query_params(params)))
url = parsed_url.to_uri().to_text().encode('ascii')
# Convert headers dictionary to
# twisted raw headers format.
headers = kwargs.get('headers')
if headers:
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.items():
if isinstance(v, (bytes, unicode)):
h.addRawHeader(k, v)
elif isinstance(v, list):
h.setRawHeaders(k, v)
headers = h
else:
headers = Headers({})
# Here we choose a right producer
# based on the parameters passed in.
bodyProducer = None
data = kwargs.get('data')
files = kwargs.get('files')
# since json=None needs to be serialized as 'null', we need to
# explicitly check kwargs for this key
has_json = 'json' in kwargs
if files:
# If the files keyword is present we will issue a
# multipart/form-data request as it suits better for cases
# with files and/or large objects.
files = list(_convert_files(files))
boundary = str(uuid.uuid4()).encode('ascii')
headers.setRawHeaders(
b'content-type',
[b'multipart/form-data; boundary=' + boundary])
if data:
data = _convert_params(data)
else:
data = []
bodyProducer = multipart.MultiPartProducer(data + files,
boundary=boundary)
elif data:
# Otherwise stick to x-www-form-urlencoded format
# as it's generally faster for smaller requests.
if isinstance(data, (dict, list, tuple)):
headers.setRawHeaders(b'content-type',
[b'application/x-www-form-urlencoded'])
data = urlencode(data, doseq=True)
bodyProducer = self._data_to_body_producer(data)
elif has_json:
# If data is sent as json, set Content-Type as 'application/json'
headers.setRawHeaders(b'content-type',
[b'application/json; charset=UTF-8'])
content = kwargs['json']
json = json_dumps(content, separators=(u',', u':')).encode('utf-8')
bodyProducer = self._data_to_body_producer(json)
cookies = kwargs.get('cookies', {})
if not isinstance(cookies, CookieJar):
cookies = cookiejar_from_dict(cookies)
cookies = merge_cookies(self._cookiejar, cookies)
wrapped_agent = CookieAgent(self._agent, cookies)
if kwargs.get('allow_redirects', True):
if kwargs.get('browser_like_redirects', False):
wrapped_agent = BrowserLikeRedirectAgent(wrapped_agent)
else:
wrapped_agent = RedirectAgent(wrapped_agent)
wrapped_agent = ContentDecoderAgent(wrapped_agent,
[(b'gzip', GzipDecoder)])
auth = kwargs.get('auth')
if auth:
wrapped_agent = add_auth(wrapped_agent, auth)
d = wrapped_agent.request(method,
url,
headers=headers,
bodyProducer=bodyProducer)
timeout = kwargs.get('timeout')
if timeout:
delayedCall = default_reactor(kwargs.get('reactor')).callLater(
timeout, d.cancel)
def gotResult(result):
if delayedCall.active():
delayedCall.cancel()
return result
d.addBoth(gotResult)
if not kwargs.get('unbuffered', False):
d.addCallback(_BufferedResponse)
return d.addCallback(_Response, cookies)
def test_multiple_proxys(self):
headers = Headers({
b'X-Forwarded-For': [b'10.1.2.3, 10.1.2.4'],
})
self.assertEqual(parse_x_forwarded_for(headers),
(['10.1.2.3', 0], None))
def test_original_proto(self):
headers = Headers({})
self.assertEqual(
parse_x_forwarded_for(headers, original_scheme='http'),
(None, 'http'))
def worker():
for i in range(numEvents):
event = makeEvent(i, 1, 0)
yield agent.request('PUT', '%s%s%d.ics' % (uri, cal, i),
Headers({"content-type": ["text/calendar"]}),
StringProducer(event))
def _send_request(
self,
request,
retry_on_dns_fail=True,
timeout=None,
long_retries=False,
ignore_backoff=False,
backoff_on_404=False,
):
"""
Sends a request to the given server.
Args:
request (MatrixFederationRequest): details of request to be sent
timeout (int|None): number of milliseconds to wait for the response headers
(including connecting to the server). 60s by default.
ignore_backoff (bool): true to ignore the historical backoff data
and try the request anyway.
backoff_on_404 (bool): Back off if we get a 404
Returns:
Deferred[twisted.web.client.Response]: resolves with the HTTP
response object on success.
Raises:
HttpResponseException: If we get an HTTP response code >= 300
(except 429).
NotRetryingDestination: If we are not yet ready to retry this
server.
FederationDeniedError: If this destination is not on our
federation whitelist
RequestSendFailed: If there were problems connecting to the
remote, due to e.g. DNS failures, connection timeouts etc.
"""
if timeout:
_sec_timeout = timeout / 1000
else:
_sec_timeout = self.default_timeout
if (self.hs.config.federation_domain_whitelist is not None
and request.destination
not in self.hs.config.federation_domain_whitelist):
raise FederationDeniedError(request.destination)
limiter = yield synapse.util.retryutils.get_retry_limiter(
request.destination,
self.clock,
self._store,
backoff_on_404=backoff_on_404,
ignore_backoff=ignore_backoff,
)
method_bytes = request.method.encode("ascii")
destination_bytes = request.destination.encode("ascii")
path_bytes = request.path.encode("ascii")
if request.query:
query_bytes = encode_query_args(request.query)
else:
query_bytes = b""
headers_dict = {
b"User-Agent": [self.version_string_bytes],
}
with limiter:
# XXX: Would be much nicer to retry only at the transaction-layer
# (once we have reliable transactions in place)
if long_retries:
retries_left = MAX_LONG_RETRIES
else:
retries_left = MAX_SHORT_RETRIES
url_bytes = urllib.parse.urlunparse((
b"matrix",
destination_bytes,
path_bytes,
None,
query_bytes,
b"",
))
url_str = url_bytes.decode('ascii')
url_to_sign_bytes = urllib.parse.urlunparse((
b"",
b"",
path_bytes,
None,
query_bytes,
b"",
))
while True:
try:
json = request.get_json()
if json:
headers_dict[b"Content-Type"] = [b"application/json"]
auth_headers = self.build_auth_headers(
destination_bytes,
method_bytes,
url_to_sign_bytes,
json,
)
data = encode_canonical_json(json)
producer = QuieterFileBodyProducer(
BytesIO(data),
cooperator=self._cooperator,
)
else:
producer = None
auth_headers = self.build_auth_headers(
destination_bytes,
method_bytes,
url_to_sign_bytes,
)
headers_dict[b"Authorization"] = auth_headers
logger.info(
"{%s} [%s] Sending request: %s %s; timeout %fs",
request.txn_id,
request.destination,
request.method,
url_str,
_sec_timeout,
)
try:
with Measure(self.clock, "outbound_request"):
# we don't want all the fancy cookie and redirect handling
# that treq.request gives: just use the raw Agent.
request_deferred = self.agent.request(
method_bytes,
url_bytes,
headers=Headers(headers_dict),
bodyProducer=producer,
)
request_deferred = timeout_deferred(
request_deferred,
timeout=_sec_timeout,
reactor=self.hs.get_reactor(),
)
response = yield request_deferred
except DNSLookupError as e:
raise_from(
RequestSendFailed(e, can_retry=retry_on_dns_fail),
e)
except Exception as e:
logger.info("Failed to send request: %s", e)
raise_from(RequestSendFailed(e, can_retry=True), e)
logger.info(
"{%s} [%s] Got response headers: %d %s",
request.txn_id,
request.destination,
response.code,
response.phrase.decode('ascii', errors='replace'),
)
if 200 <= response.code < 300:
pass
else:
# :'(
# Update transactions table?
d = treq.content(response)
d = timeout_deferred(
d,
timeout=_sec_timeout,
reactor=self.hs.get_reactor(),
)
try:
body = yield make_deferred_yieldable(d)
except Exception as e:
# Eh, we're already going to raise an exception so lets
# ignore if this fails.
logger.warn(
"{%s} [%s] Failed to get error response: %s %s: %s",
request.txn_id,
request.destination,
request.method,
url_str,
_flatten_response_never_received(e),
)
body = None
e = HttpResponseException(response.code,
response.phrase, body)
# Retry if the error is a 429 (Too Many Requests),
# otherwise just raise a standard HttpResponseException
if response.code == 429:
raise_from(RequestSendFailed(e, can_retry=True), e)
else:
raise e
break
except RequestSendFailed as e:
logger.warn(
"{%s} [%s] Request failed: %s %s: %s",
request.txn_id,
request.destination,
request.method,
url_str,
_flatten_response_never_received(e.inner_exception),
)
if not e.can_retry:
raise
if retries_left and not timeout:
if long_retries:
delay = 4**(MAX_LONG_RETRIES + 1 - retries_left)
delay = min(delay, 60)
delay *= random.uniform(0.8, 1.4)
else:
delay = 0.5 * 2**(MAX_SHORT_RETRIES - retries_left)
delay = min(delay, 2)
delay *= random.uniform(0.8, 1.4)
logger.debug(
"{%s} [%s] Waiting %ss before re-sending...",
request.txn_id,
request.destination,
delay,
)
yield self.clock.sleep(delay)
retries_left -= 1
else:
raise
except Exception as e:
logger.warn(
"{%s} [%s] Request failed: %s %s: %s",
request.txn_id,
request.destination,
request.method,
url_str,
_flatten_response_never_received(e),
)
raise
defer.returnValue(response)
def test_rawHeadersTypeChecking(self):
"""
L{Headers.setRawHeaders} requires values to be of type list.
"""
h = Headers()
self.assertRaises(TypeError, h.setRawHeaders, b'key', {b'Foo': b'bar'})
def upload(self):
"""
Upload the workflow bundle to VelesForge. Requires write access.
"""
try:
metadata = self._parse_metadata(self.path)
except Exception as e:
self.exception("Failed to upload %s:", self.path)
self.stop(Failure(e), False)
return
for key in REQUIRED_MANIFEST_FIELDS:
if key not in metadata:
raise ValueError("No \"%s\" in %s" %
(key, root.common.forge.manifest))
requires = metadata["requires"]
validate_requires(requires)
vreqfound = False
for req in requires:
if Requirement.parse(req).project_name == veles.__name__:
vreqfound = True
break
if not vreqfound:
velreq = veles.__name__ + ">=" + veles.__version__
self.warning("No VELES core requirement was specified. "
"Appended %s", velreq)
requires.append(velreq)
metadata["version"] = self.version
name = metadata["name"]
workflow = metadata["workflow"]
config = metadata["configuration"]
extra = metadata.get("files", [])
if "image" in metadata:
extra.append(metadata["image"])
files = sorted({workflow, config,
root.common.forge.manifest}.union(extra))
self.info("Uploading %s...", name)
agent = Agent(reactor)
headers = Headers({b'User-Agent': [b'twisted']})
# We will send the following:
# 4 bytes with length of metadata in JSON format
# metadata in JSON format
# tar.gz package
@implementer(IBodyProducer)
class ForgeBodyProducer(object):
def __init__(self, owner):
self.owner = owner
self.writer = None
self.length = UNKNOWN_LENGTH
self.finished = Deferred()
self.consumer = None
self.memory = 0
self.event = threading.Event()
def startProducing(self, consumer):
metabytes = json.dumps(
metadata, sort_keys=True).encode('UTF-8')
self.owner.debug("Metadata size is %d", len(metabytes))
consumer.write(struct.pack("!I", len(metabytes)))
consumer.write(metabytes)
self.consumer = consumer
self.writer.start()
return self.finished
def pauseProducing(self):
pass
def resumeProducing(self):
pass
def stopProducing(self):
if self.writer.is_alive():
self.writer.join()
def send(self, data):
self.consumer.write(data)
self.memory -= len(data)
self.event.set()
def write(self, data):
self.memory += len(data)
while self.memory > ForgeClient.UPLOAD_PENDING_BUFFERS * \
ForgeClient.UPLOAD_TAR_BUFFER_SIZE:
self.owner.debug("Suspended tar pipeline")
self.event.wait()
self.event.clear()
self.owner.debug("Scheduling send(%d bytes), pending %d bytes",
len(data), self.memory - len(data))
reactor.callFromThread(self.send, data)
def close(self):
self.owner.debug("Closing, %d bytes are pending", self.memory)
reactor.callFromThread(self.finished.callback, None)
body = ForgeBodyProducer(self)
def write_package():
tbs = ForgeClient.UPLOAD_TAR_BUFFER_SIZE
with TarFile.open(mode="w|gz", fileobj=body, bufsize=tbs,
dereference=True) as tar:
for file in files:
self.debug("Sending %s", file)
ti = TarInfo(file)
fp = os.path.join(self.path, file)
ti.size = os.path.getsize(fp)
ti.mode = 0o666
with open(fp, "rb") as fd:
tar.addfile(ti, fileobj=fd)
body.close()
writer = threading.Thread(target=write_package)
body.writer = writer
def finished(response):
self.debug("Response from server: %s", response.code)
if response.code != 200:
message = "Response code is %d" % response.code
self.error(message)
self.stop(Failure(Exception(message)), False)
else:
self.stop()
def failed(failure):
if not hasattr(failure.value, "reasons"):
try:
failure.raiseException()
except:
self.exception("Failed to upload %s:", name)
else:
self.error("Failed to upload %s:\n%s", name,
failure.value.reasons[0].getTraceback())
self.stop(failure, False)
url = self.base + root.common.forge.upload_name + "?token=" + self.id
self.debug("Sending the request to %s", url)
d = agent.request(
b'POST', url.encode('charmap'), headers=headers, bodyProducer=body)
d.addCallback(finished)
d.addErrback(failed)
def handler():
agent = Agent(reactor, pool=pnconn_pool)
if options.data is not None:
body = FileBodyProducer(StringIO(options.data))
else:
body = None
request = agent.request(options.method_string, url,
Headers(headers), body)
def received(response):
finished = Deferred()
response.deliverBody(PubNubResponse(finished, response.code))
return finished
def success(response, req_url, request):
parsed_url = urlparse(req_url)
query = parse_qs(parsed_url.query)
uuid = None
auth_key = None
if 'uuid' in query and len(query['uuid']) > 0:
uuid = query['uuid'][0]
if 'auth_key' in query and len(query['auth_key']) > 0:
auth_key = query['auth_key'][0]
response_body = response.body
code = response.code
d = Deferred()
response_info = ResponseInfo(
status_code=response.code,
tls_enabled='https' == parsed_url.scheme,
origin=parsed_url.netloc,
uuid=uuid,
auth_key=auth_key,
client_request=request)
if code != 200:
if code == 403:
status_category = PNStatusCategory.PNAccessDeniedCategory
elif code == 400:
status_category = PNStatusCategory.PNBadRequestCategory
else:
status_category = self
if code >= 500:
error = PNERR_SERVER_ERROR
else:
error = PNERR_CLIENT_ERROR
else:
error = None
status_category = PNStatusCategory.PNAcknowledgmentCategory
try:
data = json.loads(response_body)
except ValueError:
try:
data = json.loads(response_body.decode("utf-8"))
except ValueError:
raise PubNubTwistedException(
result=create_response(None),
status=create_status_response(
status_category, response_info,
PubNubException(
pn_error=PNERR_JSON_DECODING_FAILED,
errormsg='json decode error')))
if error:
raise PubNubTwistedException(
result=data,
status=create_status_response(
status_category, data, response_info,
PubNubException(errormsg=data,
pn_error=error,
status_code=response.code)))
envelope = TwistedEnvelope(
create_response(data),
create_status_response(status_category, response,
response_info, error), data)
d.callback(envelope)
return d
def failed(failure):
raise PubNubTwistedException(
result=None,
status=create_status_response(
PNStatusCategory.PNTLSConnectionFailedCategory, None,
None,
PubNubException(errormsg=str(failure),
pn_error=PNERR_CONNECTION_ERROR,
status_code=0)))
request.addErrback(failed)
request.addCallback(received)
request.addCallback(success, url, request)
return request
def open_events_socket(self, _):
agent = Agent(reactor, pool=self.connection_pool)
return agent.request('GET', 'http://localhost:%s/events' % self.session.config.get_http_api_port(),
Headers({'User-Agent': ['Tribler ' + version_id]}), None)\
.addCallback(self.on_event_socket_opened)
def httpRequest(agent,
url,
values=None,
headers=None,
method='POST',
token=None,
saveto=None):
if values is None:
values = {}
if headers is None:
headers = {}
data = ''
mtime = None
if values:
data = urllib.urlencode(values)
headers['Content-Type'] = ['application/x-www-form-urlencoded']
isfile = os.path.isfile
if saveto is not None and isfile(saveto):
# TODO - I think we need a force parameter, because we might have a
# malformed file. Or maybe just remove the file if sanity check does
# not pass.
mtime = get_mtime(saveto)
if mtime is not None:
headers['if-modified-since'] = [mtime]
if token:
headers['Authorization'] = ['Token token="%s"' % (bytes(token))]
def handle_response(response):
log.debug("RESPONSE %s %s %s" % (method, response.code, url))
if response.code == 204:
d = defer.succeed('')
elif response.code == 401:
raise Forbidden()
if saveto and mtime and response.code == 304:
log.debug('304 (Not modified): %s' % url)
raise Unchanged()
else:
class SimpleReceiver(protocol.Protocol):
def __init__(s, d):
s.buf = ''
s.d = d
def dataReceived(s, data):
s.buf += data
def connectionLost(s, reason):
# TODO: test if reason is twisted.web.client.ResponseDone,
# if not, do an errback
s.d.callback(s.buf)
d = defer.Deferred()
response.deliverBody(SimpleReceiver(d))
return d
def passthru(failure):
failure.trap(Unchanged, Forbidden)
d = agent.request(method, url, Headers(headers),
StringProducer(data) if data else None)
d.addCallback(handle_response)
if saveto:
d.addCallback(lambda body: _write_to_file(body, saveto))
d.addErrback(passthru)
return d
def test_address_only(self):
headers = Headers({
b'X-Forwarded-For': [b'10.1.2.3'],
})
self.assertEqual(parse_x_forwarded_for(headers),
(['10.1.2.3', 0], None))
def test_whenNoBasicAuthThen401(self):
d = self.agent.request('PUT', 'http://localhost:8068/', Headers({}),
None)
return d.addCallback(self._checkResponseCode, 401)
def test_v6_address(self):
headers = Headers({
b'X-Forwarded-For': [b'1043::a321:0001, 10.0.5.6'],
})
self.assertEqual(parse_x_forwarded_for(headers),
(['1043::a321:0001', 0], None))
def test_whenAccessToRootResourceThen501(self):
# Twisted gives 405 for POST but 501 for PUT???
d = self.agent.request(
'PUT', 'http://localhost:8068/',
Headers({'Authorization': ['Basic %s' % self.basic]}), None)
return d.addCallback(self._checkResponseCode, 501)
def test_original_addr(self):
headers = Headers({})
self.assertEqual(
parse_x_forwarded_for(headers, original_addr=['127.0.0.1', 80]),
(['127.0.0.1', 80], None))
def test_whenAccessToResourceChildThen400(self):
d = self.agent.request(
'PUT', 'http://localhost:8068/' + self.db + '/res.partner/4/abc',
Headers({'Authorization': ['Basic %s' % self.basic]}), None)
return d.addCallback(self._checkResponseCode, 400)
def test_no_original(self):
headers = Headers({})
self.assertEqual(parse_x_forwarded_for(headers), (None, None))
def test_whenAccessToNonExistingResourceThen404(self):
# TODO: make sure that we actually have an non-existing resource
d = self.agent.request(
'PUT', 'http://localhost:8068/' + self.db + '/res.partner/-1',
Headers({'Authorization': ['Basic %s' % self.basic]}), None)
return d.addCallback(self._checkResponseCode, 404)
def _headers(self, content_type):
if content_type is None:
return Headers()
return Headers({"content-type": [content_type]})
def test_whenAccessToAnotherNonExistingResourceThen404(self):
d = self.agent.request(
'PUT',
'http://localhost:8068/' + self.db + '/res.partner/100000000',
Headers({'Authorization': ['Basic %s' % self.basic]}), None)
return d.addCallback(self._checkResponseCode, 404)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
:param method: HTTP method (GET/POST/etc).
:type method: bytes
:param uri: Absolute URI to be retrieved.
:type uri: bytes
:param headers: HTTP headers to send with the request, or None to
send no extra headers.
:type headers: twisted.web.http_headers.Headers, None
:param bodyProducer: An object which can generate bytes to make up the
body of this request (for example, the properly encoded contents of
a file for a file upload). Or None if the request is to have
no body.
:type bodyProducer: twisted.web.iweb.IBodyProducer, None
:returns a deferred that fires when the header of the response has
been received (regardless of the response status code). Fails if
there is any problem which prevents that response from being received
(including problems that prevent the request from being sent).
:rtype: Deferred[twisted.web.iweb.IResponse]
"""
parsed_uri = URI.fromBytes(uri, defaultPort=-1)
res = yield self._route_matrix_uri(parsed_uri)
# set up the TLS connection params
#
# XXX disabling TLS is really only supported here for the benefit of the
# unit tests. We should make the UTs cope with TLS rather than having to make
# the code support the unit tests.
if self._tls_client_options_factory is None:
tls_options = None
else:
tls_options = self._tls_client_options_factory.get_options(
res.tls_server_name.decode("ascii")
)
# make sure that the Host header is set correctly
if headers is None:
headers = Headers()
else:
headers = headers.copy()
if not headers.hasHeader(b'host'):
headers.addRawHeader(b'host', res.host_header)
class EndpointFactory(object):
@staticmethod
def endpointForURI(_uri):
ep = LoggingHostnameEndpoint(
self._reactor, res.target_host, res.target_port,
)
if tls_options is not None:
ep = wrapClientTLS(tls_options, ep)
return ep
agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(), self._pool)
res = yield agent.request(method, uri, headers, bodyProducer)
defer.returnValue(res)
def my_getPage(self, p_url):
l_d = Agent(reactor).request('GET', p_url,
Headers({'User-Agent': ['twisted']}),
None)
l_d.addCallbacks(self.handleResponse, self.handleError)
return l_d
def test_getRawHeadersNoDefault(self):
"""
L{Headers.getRawHeaders} returns L{None} if the header is not found and
no default is specified.
"""
self.assertIsNone(Headers().getRawHeaders("test"))
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
self.type, rest = splittype(self.uri)
self.host, rest = splithost(rest)
def test_rawHeadersTypeCheckingValuesIterable(self):
"""
L{Headers.setRawHeaders} requires values to be of type list.
"""
h = Headers()
self.assertRaises(TypeError, h.setRawHeaders, b"key", {b"Foo": b"bar"})
def requestMock(path, method=b"GET", host=b"localhost", port=8080,
isSecure=False, body=None, headers=None):
if not headers:
headers = {}
if not body:
body = b''
request = server.Request(DummyChannel(), False)
request.site = Mock(server.Site)
request.gotLength(len(body))
request.content = BytesIO()
request.content.write(body)
request.content.seek(0)
request.requestHeaders = Headers(headers)
request.setHost(host, port, isSecure)
request.uri = path
request.prepath = []
request.postpath = path.split(b'/')[1:]
request.method = method
request.clientproto = b'HTTP/1.1'
request.setHeader = Mock(wraps=request.setHeader)
request.setResponseCode = Mock(wraps=request.setResponseCode)
request._written = BytesIO()
request.finishCount = 0
request.writeCount = 0
def registerProducer(producer, streaming):
request.producer = producer
for x in range(2):
if request.producer:
request.producer.resumeProducing()
def unregisterProducer():
request.producer = None
def finish():
request.finishCount += 1
if not request.startedWriting:
request.write(b'')
if not request.finished:
request.finished = True
request._cleanup()
def write(data):
request.writeCount += 1
request.startedWriting = True
if not request.finished:
request._written.write(data)
else:
raise RuntimeError('Request.write called on a request after '
'Request.finish was called.')
def getWrittenData():
return request._written.getvalue()
request.finish = finish
request.write = write
request.getWrittenData = getWrittenData
request.registerProducer = registerProducer
request.unregisterProducer = unregisterProducer
request.processingFailed = Mock(wraps=request.processingFailed)
return request
