def test_from_responsecaplarge(self):
response = mock.NonCallableMagicMock(spec=requests.Response)
response.headers = {
'content-type': 'Multipart/Related; boundary="samp1"'
}
cnt = io.BytesIO()
cnt.write(b'\r\n--samp1\r\n')
cnt.write(b'Header-1: Header-Value-1\r\n')
cnt.write(b'Header-2: Header-Value-2\r\n')
cnt.write(b'\r\n')
cnt.write(b'Body 1, Line 1\r\n')
cnt.write(b'Body 1, Line 2\r\n')
cnt.write(b'--samp1\r\n')
cnt.write(b'\r\n')
cnt.write(b'Body 2, Line 1\r\n')
cnt.write(b'--samp1--\r\n')
response.content = cnt.getvalue()
decoder_2 = MultipartDecoder.from_response(response)
assert decoder_2.content_type == response.headers['content-type']
assert (
decoder_2.parts[0].content == b'Body 1, Line 1\r\nBody 1, Line 2'
)
assert decoder_2.parts[0].headers[b'Header-1'] == b'Header-Value-1'
assert len(decoder_2.parts[1].headers) == 0
assert decoder_2.parts[1].content == b'Body 2, Line 1'
def test_from_response(self):
response = mock.NonCallableMagicMock(spec=requests.Response)
response.headers = {
'content-type': 'multipart/related; boundary="samp1"'
}
cnt = io.BytesIO()
cnt.write(b'\r\n--samp1\r\n')
cnt.write(b'Header-1: Header-Value-1\r\n')
cnt.write(b'Header-2: Header-Value-2\r\n')
cnt.write(b'\r\n')
cnt.write(b'Body 1, Line 1\r\n')
cnt.write(b'Body 1, Line 2\r\n')
cnt.write(b'--samp1\r\n')
cnt.write(b'\r\n')
cnt.write(b'Body 2, Line 1\r\n')
cnt.write(b'--samp1--\r\n')
response.content = cnt.getvalue()
decoder_2 = MultipartDecoder.from_response(response)
assert decoder_2.content_type == response.headers['content-type']
assert (
decoder_2.parts[0].content == b'Body 1, Line 1\r\nBody 1, Line 2'
)
assert decoder_2.parts[0].headers[b'Header-1'] == b'Header-Value-1'
assert len(decoder_2.parts[1].headers) == 0
assert decoder_2.parts[1].content == b'Body 2, Line 1'
def test_uploadhelper(self):
file = io.BytesIO(b"I'm a file, yes I am!")
response = ('{"got":[{"filename":"afile",'
'"fileref":"sha224-aaaabbbb"}]}')
mock = requests_mock.Mocker()
mock.post('https://example.com/perkeep/', text=response)
with mock:
fileref = self.uploadhelper.upload_file('afile', file,
'2010-01-02T10:20:30Z')
self.assertEqual(fileref, 'sha224-aaaabbbb')
decoder = MultipartDecoder(mock.last_request.body,
mock.last_request.headers['Content-Type'])
found_modtime, found_file = False, False
for part in decoder.parts:
if b'modtime' in part.headers[b'Content-Disposition']:
self.assertEqual(part.content, b'2010-01-02T10:20:30Z')
found_modtime = True
elif b'afile' in part.headers[b'Content-Disposition']:
self.assertEqual(part.content, b"I'm a file, yes I am!")
found_file = True
self.assertTrue(found_modtime,
'modtime was not found in multipart data')
self.assertTrue(found_file, 'file was not found in multipart data')
def test_rebuild_xml():
data = "\r\n".join(line.strip() for line in """
--MIME_boundary
Content-Type: application/soap+xml; charset=UTF-8
Content-Transfer-Encoding: 8bit
Content-ID: <*****@*****.**>
<soap:Envelope
xmlns:soap="http://www.w3.org/2003/05/soap-envelope"
xmlns:xop='http://www.w3.org/2004/08/xop/include'
xmlns:xop-mime='http://www.w3.org/2005/05/xmlmime'>
<soap:Body>
<submitClaim>
<accountNumber>5XJ45-3B2</accountNumber>
<eventType>accident</eventType>
<image xop-mime:content-type='image/jpeg'><xop:Include href="cid:[email protected]"/></image>
</submitClaim>
</soap:Body>
</soap:Envelope>
--MIME_boundary
Content-Type: image/jpeg
Content-Transfer-Encoding: binary
Content-ID: <*****@*****.**>
...binary JPG image...
--MIME_boundary--
""".splitlines()).encode("utf-8")
response = stub(
status_code=200,
content=data,
encoding=None,
headers={
"Content-Type":
'multipart/related; boundary=MIME_boundary; type="application/soap+xml"; start="<*****@*****.**>" 1'
},
)
decoder = MultipartDecoder(response.content,
response.headers["Content-Type"], "utf-8")
document = etree.fromstring(decoder.parts[0].content)
message_pack = MessagePack(parts=decoder.parts[1:])
xop.process_xop(document, message_pack)
expected = """
<soap:Envelope xmlns:soap="http://www.w3.org/2003/05/soap-envelope" xmlns:xop="http://www.w3.org/2004/08/xop/include" xmlns:xop-mime="http://www.w3.org/2005/05/xmlmime">
<soap:Body>
<submitClaim>
<accountNumber>5XJ45-3B2</accountNumber>
<eventType>accident</eventType>
<image xop-mime:content-type="image/jpeg">Li4uYmluYXJ5IEpQRyBpbWFnZS4uLg==</image>
</submitClaim>
</soap:Body>
</soap:Envelope>
"""
assert_nodes_equal(etree.tostring(document), expected)
def process_reply(self, client, operation, response):
"""Process the XML reply from the server.
:param client: The client with which the operation was called
:type client: zeep.client.Client
:param operation: The operation object from which this is a reply
:type operation: zeep.wsdl.definitions.Operation
:param response: The response object returned by the remote server
:type response: requests.Response
"""
if response.status_code != 200 and not response.content:
raise TransportError(
u'Server returned HTTP status %d (no content available)'
% response.status_code)
content_type = response.headers.get('Content-Type', 'text/xml')
media_type = get_media_type(content_type)
message_pack = None
if media_type == 'multipart/related':
decoder = MultipartDecoder(
response.content, content_type, response.encoding or 'utf-8')
content = decoder.parts[0].content
if len(decoder.parts) > 1:
message_pack = MessagePack(parts=decoder.parts[1:])
else:
content = response.content
try:
doc = parse_xml(
content, self.transport,
strict=client.wsdl.strict,
xml_huge_tree=client.xml_huge_tree)
except XMLSyntaxError:
raise TransportError(
u'Server returned HTTP status %d (%s)'
% (response.status_code, response.content))
if client.wsse:
client.wsse.verify(doc)
doc, http_headers = plugins.apply_ingress(
client, doc, response.headers, operation)
# If the response code is not 200 or if there is a Fault node available
# then assume that an error occured.
fault_node = doc.find(
'soap-env:Body/soap-env:Fault', namespaces=self.nsmap)
if response.status_code != 200 or fault_node is not None:
return self.process_error(doc, operation)
result = operation.process_reply(doc)
if message_pack:
message_pack._set_root(result)
return message_pack
return result
def test_file_is_being_encoded():
"""Test if file is being encoded."""
data = bytes([0x13] * 1024)
stream = StreamWrapper(BytesIO(data), 1024)
form = _multipart_encoder('test', stream, 'application/x-binary')
response_data = form.read()
decoder = MultipartDecoder(response_data, form.content_type)
assert decoder.parts[0].content == data
def setUp(self):
self.sample_1 = (
('field 1', 'value 1'),
('field 2', 'value 2'),
('field 3', 'value 3'),
('field 4', 'value 4'),
)
self.boundary = 'test boundary'
self.encoded_1 = MultipartEncoder(self.sample_1, self.boundary)
self.decoded_1 = MultipartDecoder(self.encoded_1.to_string(),
self.encoded_1.content_type)
def UnpackMultipartRequest(environ) :
"""Unpack a multipart request
"""
request_body_size = int(environ.get('CONTENT_LENGTH', 0))
request_type = environ.get('CONTENT_TYPE','')
if not request_type.startswith('multipart/form-data') :
msg = 'unknown request type, <{0}>'.format(request_type)
raise Exception(msg)
request_body = environ['wsgi.input'].read(request_body_size)
return MultipartDecoder(request_body, request_type)
def _eval_adhoc(self, lang, line, cell):
self._check_for_session()
valid_lang = ("xquery", "javascript")
if lang not in valid_lang:
raise TypeError("lang can only be " + ", ".join(valid_lang))
args = shlex.split(line, comments = True)
self._check_invalid_args(args, ("-raw", "-dataframe"))
output_opt = "default"
if "-raw" in args:
output_opt = "raw"
if "-dataframe" in args:
output_opt = "dataframe"
# execute code on MarkLogic
url = self.session_baseurl + "/v1/eval"
body = {
lang: cell
}
output = []
response = self.session.post(url, data = body)
if response.status_code == 200:
headers = { header.lower(): value for (header, value) in response.headers.items() }
# as per docs, /v1/eval should always return multipart/mixed
if "content-type" in headers and headers["content-type"].startswith("multipart/mixed"):
mp_data = MultipartDecoder.from_response(response)
for part in mp_data.parts:
part_headers = { header.decode("utf-8").lower(): value.decode("utf-8") for (header, value) in part.headers.items() }
part_content_type = part_headers["content-type"]
part_x_primitive = part_headers["x-primitive"]
part_content = part.content.decode("utf-8")
data = part_content if output_opt == "raw" else self._parse_content(part_content, part_content_type, part_x_primitive)
output.append(data)
else:
raise EvalError("Request to %s returned %s %s: %s" % (url, response.status_code, response.reason, response.text))
output = output[0] if len(output) == 1 else output
if output_opt == "dataframe":
try:
output = pandas.DataFrame(output) # try to fit into a dataframe, may raise an exception depending on result's shape
except:
raise CoercionError("Unable to fit response into a DataFrame")
return {
"status_code": response.status_code,
"output": output,
"output_opt": output_opt,
"args": args
}
def parse_multipart(response: Response_) -> Sequence[Any]:
"""
RFC 2045 describes the format of an Internet message body containing a MIME message. The
body contains one or more body parts, each preceded by a boundary delimiter line, and the
last one followed by a closing boundary delimiter line. After its boundary delimiter line,
each body part then consists of a header area, a blank line, and a body area.
HTTP/1.1 200 OK
Server: Apache/2.0.13
Date: Fri, 22 OCT 2004 12:03:38 GMT
Cache-Control: private
RETS-Version: RETS/1.7.2
MIME-Version: 1.0
Content-Type: multipart/parallel; boundary="simple boundary"
--simple boundary
Content-Type: image/jpeg
Content-ID: 123456
Object-ID: 1
<binary data>
--simple boundary
Content-Type: text/xml
Content-ID: 123457
Object-ID: 1
<RETS ReplyCode="20403" ReplyText="There is no listing with that ListingID"/>
--simple boundary--
"""
multipart = MultipartDecoder.from_response(response, response.encoding)
# We need to decode the headers because MultipartDecoder returns bytes keys and values,
# while requests.Response.headers uses str keys and values.
for part in multipart.parts:
part.headers = _decode_headers(part.headers, response.encoding)
def parse_multipart(parts):
for part in parts:
try:
yield parse_response(part)
except RetsApiError as e:
if e.reply_code != 20403: # No object found
raise
return tuple(parse_multipart(multipart.parts))
def _parse_multipart(response: ResponseLike,
parser: Parser) -> Sequence[Object]:
"""
RFC 2045 describes the format of an Internet message body containing a MIME message. The
body contains one or more body parts, each preceded by a boundary delimiter line, and the
last one followed by a closing boundary delimiter line. After its boundary delimiter line,
each body part then consists of a header area, a blank line, and a body area.
HTTP/1.1 200 OK
Server: Apache/2.0.13
Date: Fri, 22 OCT 2004 12:03:38 GMT
Cache-Control: private
RETS-Version: RETS/1.7.2
MIME-Version: 1.0
Content-Type: multipart/parallel; boundary="simple boundary"
--simple boundary
Content-Type: image/jpeg
Content-ID: 123456
Object-ID: 1
<binary data>
--simple boundary
Content-Type: text/xml
Content-ID: 123457
Object-ID: 1
<RETS ReplyCode="20403" ReplyText="There is no listing with that ListingID"/>
--simple boundary--
"""
encoding = response.encoding or DEFAULT_ENCODING
multipart = MultipartDecoder.from_response(response, encoding)
# We need to decode the headers because MultipartDecoder returns bytes keys and values,
# while requests.Response.headers uses str keys and values.
for part in multipart.parts:
part.headers = _decode_headers(part.headers, encoding)
objects = (_parse_body_part(part, parser) for part in multipart.parts)
return tuple(object_ for object_ in objects if object_ is not None)
def get_blocks(self, block_ids):
"""Get a list of blocks from the storage service using the get blocks
interface, block ids should be base64 encoded
:param block_ids: list of base64 encoded strings
:returns list of strings: list of byte strings containing block data
"""
request_identifier = self.request_identifier
request_headers = {
'x-request-identifier': 'request{0}'.format(request_identifier)
}
url = "{0}/block/gets".format(self.ServiceURL)
try:
response = self.session.post(url,
json=block_ids,
timeout=self.default_timeout)
except Exception as e:
logger.warn('unknown exception (get_blocks); %s', str(e))
raise StorageException(str(e)) from e
response.raise_for_status()
try:
# decode the response, note that we put the multipart encoding into a
# customer header because urllib3 doesn't seem to like requests.toolbelt's
# multipart encoding
mp_decoder = MultipartDecoder(response.content,
response.headers['x-content-type'])
block_data_list = []
for part in mp_decoder.parts:
block_data_list.append(part.content)
return block_data_list
except Exception as e:
logger.warn('unknown exception (get_blocks); %s', str(e))
raise StorageException(str(e)) from e
def process_reply(self, client, operation, response):
"""Process the XML reply from the server.
:param client: The client with which the operation was called
:type client: zeep.client.Client
:param operation: The operation object from which this is a reply
:type operation: zeep.wsdl.definitions.Operation
:param response: The response object returned by the remote server
:type response: requests.Response
"""
if response.status_code in (201, 202) and not response.content:
return None
elif response.status_code != 200 and not response.content:
raise TransportError(
u"Server returned HTTP status %d (no content available)"
% response.status_code,
status_code=response.status_code,
)
content_type = response.headers.get("Content-Type", "text/xml")
media_type = get_media_type(content_type)
message_pack = None
# If the reply is a multipart/related then we need to retrieve all the
# parts
if media_type == "multipart/related":
decoder = MultipartDecoder(
response.content, content_type, response.encoding or "utf-8"
)
content = decoder.parts[0].content
if len(decoder.parts) > 1:
message_pack = MessagePack(parts=decoder.parts[1:])
else:
content = response.content
try:
doc = parse_xml(content, self.transport, settings=client.settings)
except XMLSyntaxError as exc:
raise TransportError(
"Server returned response (%s) with invalid XML: %s.\nContent: %r"
% (response.status_code, exc, response.content),
status_code=response.status_code,
content=response.content,
)
# Check if this is an XOP message which we need to decode first
if message_pack:
if process_xop(doc, message_pack):
message_pack = None
if client.wsse:
client.wsse.verify(doc)
doc, http_headers = plugins.apply_ingress(
client, doc, response.headers, operation
)
# If the response code is not 200 or if there is a Fault node available
# then assume that an error occured.
fault_node = doc.find("soap-env:Body/soap-env:Fault", namespaces=self.nsmap)
if response.status_code != 200 or fault_node is not None:
return self.process_error(doc, operation)
result = operation.process_reply(doc)
if message_pack:
message_pack._set_root(result)
return message_pack
return result
def test_non_multipart_response_fails(self):
jpeg_response = mock.NonCallableMagicMock(spec=requests.Response)
jpeg_response.headers = {'content-type': 'image/jpeg'}
with pytest.raises(NonMultipartContentTypeException):
MultipartDecoder.from_response(jpeg_response)
def test_non_multipart_response_fails(self):
jpeg_response = mock.NonCallableMagicMock(spec=requests.Response)
jpeg_response.headers = {'content-type': 'image/jpeg'}
with pytest.raises(NonMultipartContentTypeException):
MultipartDecoder.from_response(jpeg_response)