def parse_multipart(request):
content_type = request.headers.get("content-type")
if content_type is None or not content_type.startswith(
"multipart/related"):
utils.error.invalid("Content-type header in multipart upload", None)
_, _, boundary = content_type.partition("boundary=")
if boundary is None:
utils.error.missing(
"boundary in content-type header in multipart upload", None)
body = extract_media(request)
try:
decoder = MultipartDecoder(body, content_type)
except ImproperBodyPartContentException as e:
utils.error.invalid("Multipart body is malformed\n%s" % str(body),
None)
if len(decoder.parts) != 2:
utils.error.invalid("Multipart body is malformed\n%s" % str(body),
None)
resource = decoder.parts[0].text
metadata = json.loads(resource)
content_type_key = "content-type".encode("utf-8")
headers = decoder.parts[1].headers
content_type = {"content-type": headers[content_type_key].decode("utf-8")}
media = decoder.parts[1].content
return metadata, content_type, media
def decode_runs_post_form(event):
"""Convert the multipart form that comes with a POST to /runs into a
dictionary.
"""
# Decode the form data
content_type = event["headers"].get("Content-Type") or event["headers"].get("content-type")
decoder = MultipartDecoder(event["body"].encode(), content_type)
decoded_body = {}
for part in decoder.parts:
# parts.headers doesn't seems to support key lookup??? So just iterate
# through the values, there are usually two
for key, value in part.headers.items():
if key == b"Content-Disposition":
content_disposition = cgi.parse_header(value.decode())
name = content_disposition[1]["name"]
filename = content_disposition[1].get("filename")
break
# I guess it's okay to fail if content-disposition doesn't have a name
# I think that means the form isn't right
if name == "workflow_attachment":
decoded_body.setdefault("workflow_attachment", {})[filename] = part.text
else:
decoded_body[name] = part.text
return decoded_body
def parse(self):
decoded_content = MultipartDecoder(self.content,
self.content_type_header)
key_to_content_map = {}
for part in decoded_content.parts:
cds = part.headers[b'Content-Disposition']
key = ExtractDataParser.get_key_dstring(cds)
key_to_content_map[key] = part.content
# For Json Response
keys: MultiPartKey = ExtractDataParser.get_keys(key_to_content_map)
extract_json = key_to_content_map[keys.contentAnalyzerResponse].decode()
extract_content_analyzer_res = ExtractContentAnalyzerResponse.from_json(extract_json)
extract_output_metadata: ExtractPDFOutputMetadata = extract_content_analyzer_res.outputs
# For ElementsInfo
structured_output = key_to_content_map[keys.jsonoutput].decode()
mt = mimetypes.MimeTypes()
extension = mt.guess_extension(extract_output_metadata.indexed_meta_info[keys.jsonoutput].dc_format)
structure_output: StructuredData = StructuredData(
extension,
json.loads(structured_output))
self.ed_zipper.add_structured_data(structure_output)
# For Elements Renditions
for rendition_key in keys.rendition:
rendition_output: ExtractRenditionOutput = self.frame_extract_rendition_output(
extract_output_metadata,
structure_output,
key_to_content_map[rendition_key],
rendition_key)
self.ed_zipper.add_rendition_data(rendition_output)
def uris(self, database, root=None, connection=None):
"""Get a list of all the URIs in a database.
If root is provided, only URIs that start-with() that string
will be returned.
"""
uris = []
if connection is None:
connection = self.connection
mleval = Eval(connection)
version = "xquery version '1.0-ml';"
if root is None or root == "":
mleval.set_xquery("{} cts:uris()".format(version))
else:
root = root.replace("'", "'")
mleval.set_xquery("{} cts:uris()[starts-with(.,'{}')]" \
.format(version, root))
mleval.set_database(database)
response = mleval.eval()
if 'content-type' in response.headers:
if response.headers['content-type'].startswith("multipart/mixed"):
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
uris.append(part.text)
return uris
def do_POST(self):
if not self.headers['content-length']:
self._make_reasponse(
400, message='You must provide files for transfer.')
return
file = self.rfile.read(int(self.headers['content-length']))
file_hashes = []
if 'multipart/form-data' in self.headers['content-type']:
data = MultipartDecoder(file,
content_type=self.headers['content-type'])
for part in data.parts:
pattern_matches = re.findall(
r'.*filename="(.*)".*', part.headers[
'Content-Disposition'.encode()].decode('utf-8'))
if part.headers.get('Content-Type'.encode()
) == 'application/octet-stream'.encode():
file_hashes.append(
self._save_file(file=part.content,
filename=pattern_matches[0]))
else:
self._make_reasponse(
400,
message=
'Для передачи файлов используйте content-type="multipart/form-data".'
)
self._make_reasponse(200,
message='\n'.join(
map(lambda x: str(x), file_hashes)))
def IDclaimThisTask(self, code):
self.SRmutex.acquire()
try:
response = self.session.patch(
"https://{}/ID/tasks/{}".format(self.server, code),
json={
"user": self.user,
"token": self.token
},
verify=False,
)
if response.status_code == 204:
raise PlomTakenException("Task taken by another user.")
response.raise_for_status()
except requests.HTTPError as e:
if response.status_code == 401:
raise PlomAuthenticationException() from None
else:
raise PlomSeriousException(
"Some other sort of error {}".format(e)) from None
finally:
self.SRmutex.release()
imageList = []
for img in MultipartDecoder.from_response(response).parts:
imageList.append(BytesIO(
img.content).getvalue()) # pass back image as bytes
return imageList
def acquire_bulk(changes):
postlist = []
for change in changes:
postlist.append('{{"id":"{}","rev":"{}","atts_since":[]}}'.format(
change['id'], change['changes'][0]['rev']))
payload = '{{"docs":[{}]}}'.format(','.join(postlist))
post_headers = {
"Host": 'couchbase-fallenlondon.storynexus.com:4984',
"Content-Type": "application/json",
"Connection": None,
"Accept-Encoding": None,
"Accept": None,
"User-Agent": None
}
r = requests.post(
'http://couchbase-fallenlondon.storynexus.com:4984/sync_gateway_json/_bulk_get?revs=true&attachments=true',
data=payload,
headers=post_headers)
decoder = Decoder.from_response(r)
updates = [json.loads(x.text) for x in decoder.parts]
dec = []
for u in updates:
try:
dec.append(json.loads(clean(decrypt(u['body']))))
except KeyError:
print(u)
return dec
def MrequestOriginalImages(self, task):
self.SRmutex.acquire()
try:
response = self.session.get(
"https://{}/MK/originalImages/{}".format(self.server, task),
json={
"user": self.user,
"token": self.token
},
verify=False,
)
if response.status_code == 204:
raise PlomNoMoreException("No task = {}.".format(task))
response.raise_for_status()
# response is [image1, image2,... image.n]
imageList = []
for img in MultipartDecoder.from_response(response).parts:
imageList.append(BytesIO(img.content).getvalue())
except requests.HTTPError as e:
if response.status_code == 401:
raise PlomAuthenticationException() from None
elif response.status_code == 404:
raise PlomNoMoreException(
"Cannot find image file for {}".format(task)) from None
else:
raise PlomSeriousException(
"Some other sort of error {}".format(e)) from None
finally:
self.SRmutex.release()
return imageList
def _parse_post_data(self, vector, num, name):
"""
Parse post data to verify it can be converted into an object by the auditors. This prevents conversion errors
for each check and vector combination during the scan.
:param vector: vector dictionary
:param num: index number
:param name: file name
:return: True or False
"""
headers = vector['headers']
post_data = vector['data']
# check data
# content-type checked in self._check_content_type()
if not post_data:
return True
# check json
if headers['Content-Type'].startswith(HTTP.CONTENT_TYPE.JSON):
try:
json.loads(post_data)
except JSONDecodeError:
logger.warning("'postData' is not valid JSON for vector #%d in '%s'. Ignoring.", num, name)
return False
# check multipart
if headers['Content-Type'].startswith(HTTP.CONTENT_TYPE.MULTIPART):
try:
MultipartDecoder(post_data.encode(), headers['Content-Type'])
except (ImproperBodyPartContentException, AttributeError):
logger.warning("'postData' is not valid multipart data for vector #%d in '%s'. Ignoring.", num, name)
return False
# default
return True
def IDrequestImage(self, code):
self.SRmutex.acquire()
try:
response = self.session.get(
"https://{}/ID/images/{}".format(self.server, code),
json={
"user": self.user,
"token": self.token
},
verify=False,
)
response.raise_for_status()
imageList = []
for img in MultipartDecoder.from_response(response).parts:
imageList.append(BytesIO(
img.content).getvalue()) # pass back image as bytes
except requests.HTTPError as e:
if response.status_code == 401:
raise PlomAuthenticationException() from None
elif response.status_code == 404:
raise PlomSeriousException(
"Cannot find image file for {}.".format(code)) from None
elif response.status_code == 409:
raise PlomSeriousException(
"Another user has the image for {}. This should not happen"
.format(code)) from None
else:
raise PlomSeriousException(
"Some other sort of error {}".format(e)) from None
finally:
self.SRmutex.release()
return imageList
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('AZ Function Post Request for Prediction')
try:
image_bytes = req.get_body()
#logging.info(f'{image_bytes}')
content_type = req.headers.get("Content-Type")
formdatadecoder = MultipartDecoder(image_bytes, content_type)
# assumes there is only one part (our file) being uploaded
for part in formdatadecoder.parts:
resp_body = predict(part.content)
break
return func.HttpResponse(status_code=200,
headers={"Content-Type": "application/json"},
body=json.dumps(resp_body))
except Exception as e:
raise e
logging.error(e)
return func.HttpResponse(
"Either you did not post a compatible image or an unknown error occurred.",
status_code=400)
return func.HttpResponse()
def decode_file_upload_multipart_request(
request_object_proxy: _RequestObjectProxy,
) -> DecodedFileUploadMultipartRequest:
"""
Get the decoded parts of a multipart request from a history entry of
requests_mock.request_history.
This method exists because of limitations with requests_mock:
- requests_mock.request_history[n].text is a property method that attempts utf-8
decoding on the body of the request. But we have binary data in our body (we're
uploading zip files), so the decode just fails (understandably). To workaround
this, we abuse that we have access to the request itself at
requests_mock.request_history[n]._request. Then we can access .body on that
without decoding.
- The object itself from requests_mock.request_history[n] is a private
_RequestObjectProxy. This shouldn't be so. It's returned from a public API.
Make issue requests for these when able at
https://github.com/jamielennox/requests-mock/issues ?
"""
return DecodedFileUploadMultipartRequest.from_parts(
MultipartDecoder(
content=request_object_proxy._request.body,
content_type=request_object_proxy.headers["Content-Type"],
).parts)
def uris(self, database, root=None, connection=None):
"""Get a list of all the URIs in a database.
If root is provided, only URIs that start-with() that string
will be returned.
"""
uris = []
if connection is None:
connection = self.connection
mleval = Eval(connection)
version = "xquery version '1.0-ml';"
if root is None or root == "":
mleval.set_xquery("{} cts:uris()".format(version))
else:
root = root.replace("'", "'")
mleval.set_xquery("{} cts:uris()[starts-with(.,'{}')]" \
.format(version, root))
mleval.set_database(database)
response = mleval.eval()
if 'content-type' in response.headers:
if response.headers['content-type'].startswith("multipart/mixed"):
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
uris.append(part.text)
return uris
def _get_parts():
content = request.get_data()
content_type = request.headers['content-type']
logger.info(f"Received ContentType Header ==> {content_type}")
decoder = MultipartDecoder(content, content_type)
files = []
for p in decoder.parts:
result = dict()
result['content'] = p.content
content_type = p.headers.get(b'content-type')
if content_type is not None:
result['content-type'] = content_type.decode("utf-8")
content_disposition = p.headers.get(b'content-disposition')
if content_disposition is not None:
content_disposition = content_disposition.decode("utf-8")
m = re.search(r'filename=\"([^\"]*)', content_disposition)
result['content-disposition'] = content_disposition
if m and m.group(1):
result['filename'] = m.group(1)
files.append(result)
return files
def test_pushover_success_with_imagebase64(srv, caplog):
module = load_module_from_file("mqttwarn/services/pushover.py")
image = open("./assets/pushover.png", "rb").read()
item = Item(
config={},
target="test",
addrs=["userkey2", "appkey2"],
message="⚽ Notification message ⚽",
data={"imagebase64": base64.encodebytes(image)},
)
with caplog.at_level(logging.DEBUG):
add_successful_mock_response()
outcome = module.plugin(srv, item)
# Check response status.
assert responses.calls[0].response.status_code == 200
assert responses.calls[0].response.text == '{"status": 1}'
# Decode multipart request.
request = responses.calls[0].request
decoder = MultipartDecoder(request.body, request.headers["Content-Type"])
content_disposition_headers = []
contents = {}
for part in decoder.parts:
content_disposition_headers.append(part.headers[b"Content-Disposition"])
key = part.headers[b"Content-Disposition"]
contents[key] = part.content
# Proof request has all body parts.
assert content_disposition_headers == [
b'form-data; name="user"',
b'form-data; name="token"',
b'form-data; name="retry"',
b'form-data; name="expire"',
b'form-data; name="message"',
b'form-data; name="attachment"; filename="image.jpg"',
]
# Proof parameter body parts, modulo image content, have correct values.
assert list(contents.values())[:-1] == [
b"userkey2",
b"appkey2",
b"60",
b"3600",
b"\xe2\x9a\xbd Notification message \xe2\x9a\xbd",
]
# Proof image has content.
assert len(decoder.parts[-1].content) == 45628
assert outcome is True
assert "Sending pushover notification to test" in caplog.text
assert "Successfully sent pushover notification" in caplog.text
def _download_batch(self, dataset_id, ids):
for batch_ids in batched(ids):
response = self._api.post(
'images.bulk.download', {ApiField.DATASET_ID: dataset_id, ApiField.IMAGE_IDS: batch_ids})
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
img_id = int(re.findall('name="(.*)"', part.headers[b'Content-Disposition'].decode('utf-8'))[0])
yield img_id, part
def _download_batch(self, docs, down_map):
"""Download a batch of files"""
if self.dryrun:
return
self.logger.debug("Downloading batch")
self.logger.debug(docs)
resp = docs.get()
decoder = MultipartDecoder.from_response(resp)
self.logger.debug("Downloaded {} bytes in {} parts" \
.format(len(resp.text), len(decoder.parts)))
meta_part = None
content_part = None
splitregex = re.compile(';\\s*')
if not decoder.parts:
raise RuntimeError("FAILED TO GET ANY PARTS!?")
for mimepart in decoder.parts:
disp = mimepart.headers[b'Content-Disposition'].decode('utf-8')
if 'category=metadata' in disp:
if meta_part is not None:
raise RuntimeError("More than one metadata part!?")
meta_part = mimepart
else:
content_part = mimepart
disp = content_part.headers[b'Content-Disposition'].decode(
'utf-8')
dispositions = splitregex.split(disp)
filename = None
for disp in dispositions:
if disp.startswith("filename="):
filename = disp[10:len(disp) - 1]
body_content_type = content_part.headers[
b'Content-Type'].decode('utf-8')
if filename is None:
raise RuntimeError("Multipart without filename!?")
#print("FN:",filename)
last_modified = None
stanza = down_map[filename]
if meta_part is not None:
last_modified = self._store_metadata(meta_part, down_map[filename], \
body_content_type, filename)
if last_modified is not None:
stanza['timestamp'] = last_modified
self._store_content(content_part, stanza)
meta_part = None
content_part = None
def multipart_parse(data, content_type, encoding='latin1'):
"""
parse multipart http response into headers, content tuples
:param data: bytes http multipart response body
:param content_type: str http response content-type
:param encoding: str encoding to use when decoding content
:return:
"""
decoder = MultipartDecoder(data, content_type, encoding)
return [body_part_to_headers_and_data(part) for part in decoder.parts]
def _download_batch_by_hashes(self, hashes):
for batch_hashes in batched(hashes):
response = self._api.post(
'images.bulk.download-by-hash', {ApiField.HASHES: batch_hashes})
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
content_utf8 = part.headers[b'Content-Disposition'].decode('utf-8')
# Find name="1245" preceded by a whitespace, semicolon or beginning of line.
# The regex has 2 capture group: one for the prefix and one for the actual name value.
h = content_utf8.replace("form-data; name=\"", "")[:-1]
yield h, part
def _download_batch(self, docs, down_map):
"""Download a batch of files"""
if self.dryrun:
return
self.logger.debug("Downloading batch")
self.logger.debug(docs)
resp = docs.get()
decoder = MultipartDecoder.from_response(resp)
self.logger.debug("Downloaded {} bytes in {} parts" \
.format(len(resp.text), len(decoder.parts)))
meta_part = None
content_part = None
splitregex = re.compile(';\\s*')
if not decoder.parts:
raise RuntimeError("FAILED TO GET ANY PARTS!?")
for mimepart in decoder.parts:
disp = mimepart.headers[b'Content-Disposition'].decode('utf-8')
if 'category=metadata' in disp:
if meta_part is not None:
raise RuntimeError("More than one metadata part!?")
meta_part = mimepart
else:
content_part = mimepart
disp = content_part.headers[b'Content-Disposition'].decode('utf-8')
dispositions = splitregex.split(disp)
filename = None
for disp in dispositions:
if disp.startswith("filename="):
filename = disp[10:len(disp)-1]
body_content_type = content_part.headers[b'Content-Type'].decode('utf-8')
if filename is None:
raise RuntimeError("Multipart without filename!?")
#print("FN:",filename)
last_modified = None
stanza = down_map[filename]
if meta_part is not None:
last_modified = self._store_metadata(meta_part, down_map[filename], \
body_content_type, filename)
if last_modified is not None:
stanza['timestamp'] = last_modified
self._store_content(content_part, stanza)
meta_part = None
content_part = None
def _download_batch(self, dataset_id, ids):
for batch_ids in batched(ids):
response = self._api.post(
'images.bulk.download', {ApiField.DATASET_ID: dataset_id, ApiField.IMAGE_IDS: batch_ids})
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
content_utf8 = part.headers[b'Content-Disposition'].decode('utf-8')
# Find name="1245" preceded by a whitespace, semicolon or beginning of line.
# The regex has 2 capture group: one for the prefix and one for the actual name value.
img_id = int(re.findall(r'(^|[\s;])name="(\d*)"', content_utf8)[0][1])
yield img_id, part
def parse_response(response) -> typing.Union[bytes, None]:
if response.status == http.client.NO_CONTENT:
return None
if not response.status == http.client.OK:
raise HTTPError(response=response)
parsed = MultipartDecoder(response.read(),
response.headers['content-type'][0].decode())
for part in parsed.parts:
if part.headers[b'Content-Type'] == b'application/octet-stream':
return part.content
def last_modified(self, database, uris=None, connection=None):
"""Get a list of last-modified times.
If uris are provided, returns last modified times for those URIs,
otherwise attempts to find times for all URIs in the database.
This requires the database setting to manage last modified times,
naturally.
"""
if connection is None:
connection = self.connection
mleval = Eval(connection)
if uris is None:
leturis = "let $uris := cts:uris()"
else:
leturis = "let $uris := cts:uris()["
comma = ""
for uri in uris:
leturis += "{}'{}'".format(comma, uri)
comma = ", "
leturis += "]"
lines = (leturis, \
'let $dts := for $uri in $uris', \
' let $dt := xdmp:document-get-properties($uri, xs:QName("prop:last-modified"))', \
' where $dt', \
' return', \
' object-node { "uri": $uri, "dt": string($dt) }', \
'return', \
' array-node { $dts }')
xquery = "\n".join(lines)
#print(xquery)
mleval.set_xquery(xquery)
mleval.set_database(database)
response = mleval.eval()
data = None
if 'content-type' in response.headers:
if response.headers['content-type'].startswith("multipart/mixed"):
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
if data is None:
data = json.loads(part.text)
else:
raise RuntimeError(
"Multipart reply to timestamp query!?")
return data
def last_modified(self, database, uris=None, connection=None):
"""Get a list of last-modified times.
If uris are provided, returns last modified times for those URIs,
otherwise attempts to find times for all URIs in the database.
This requires the database setting to manage last modified times,
naturally.
"""
if connection is None:
connection = self.connection
mleval = Eval(connection)
if uris is None:
leturis = "let $uris := cts:uris()"
else:
leturis = "let $uris := cts:uris()["
comma = ""
for uri in uris:
leturis += "{}'{}'".format(comma, uri)
comma = ", "
leturis += "]"
lines = (leturis, \
'let $dts := for $uri in $uris', \
' let $dt := xdmp:document-get-properties($uri, xs:QName("prop:last-modified"))', \
' where $dt', \
' return', \
' object-node { "uri": $uri, "dt": string($dt) }', \
'return', \
' array-node { $dts }')
xquery = "\n".join(lines)
#print(xquery)
mleval.set_xquery(xquery)
mleval.set_database(database)
response = mleval.eval()
data = None
if 'content-type' in response.headers:
if response.headers['content-type'].startswith("multipart/mixed"):
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
if data is None:
data = json.loads(part.text)
else:
raise RuntimeError("Multipart reply to timestamp query!?")
return data
def _get_post_multipart(self):
length = int(self.headers['Content-Length'])
logging.info("Receiving multipart message")
data = self.rfile.read(length)
with tempfile.NamedTemporaryFile(delete=False) as f:
logging.info("Saving debug data to %s" % f.name)
f.write(b"POST /api/mailgun/incoming HTTP/1.1\n")
for header in self.headers.keys():
f.write(("%s: %s\n" % (header, self.headers[header])).encode(
'utf-8', 'ignore'))
f.write(b"\n")
f.write(data)
return MultipartDecoder(data, self.headers['Content-Type'])
def _parse_multipart(self, multipart_string, content_type):
"""
Parse multipart data string and return decoder object.
:param multipart_string: multipart data string
:param content_type: content-type string
:return: Multipart decoder
"""
# parse
try:
decoder = MultipartDecoder(multipart_string.encode(), content_type)
except ImproperBodyPartContentException:
raise InvalidFormatException("Unable to parse multipart form data")
except (NonMultipartContentTypeException, AttributeError):
raise InvalidFormatException(
"Unable to parse multipart content-type")
return decoder
def parse_multipart(request):
# decode the request
decoder = MultipartDecoder(request.raw_body,
request.headers['content-type'])
body = {}
files = {}
# loop each part in the multipart upload
for p in decoder.parts:
data = p.content
meta_data = {}
# get the content type
content_type = p.headers.get(b'content-type', b'').decode()
if content_type:
meta_data['content-type'] = content_type
# parse content disposition
content_disposition = p.headers.get(b'content-disposition',
b'').decode()
if content_disposition:
meta = content_disposition.split(';')
for m in meta:
m = re.sub(r'\s+', '', m)
try:
# if there is a key value pair here, add to the meta_data for this field
k, v = m.split('=')
meta_data[k] = v.strip('"')
except:
pass
field_name = meta_data.get('name')
# if there is no content-type, just store as a string
if not meta_data.get('content-type'):
meta_data = data.decode()
multi_value_insert(body, field_name, meta_data)
if type(meta_data) is dict and data:
multi_value_insert(files, field_name, data)
return (body, files)
def download_batch(self, dataset_id, ids, paths, progress_cb=None):
id_to_path = {id: path for id, path in zip(ids, paths)}
MAX_BATCH_SIZE = 50
for batch_ids in batched(ids, MAX_BATCH_SIZE):
response = self.api.post('images.bulk.download', {
ApiField.DATASET_ID: dataset_id,
ApiField.IMAGE_IDS: batch_ids
})
decoder = MultipartDecoder.from_response(response)
for idx, part in enumerate(decoder.parts):
img_id = int(
re.findall(
'name="(.*)"',
part.headers[b'Content-Disposition'].decode('utf-8'))
[0])
with open(id_to_path[img_id], 'wb') as w:
w.write(part.content)
progress_cb(1)
def MrequestWholePaper(self, code, questionNumber=0):
self.SRmutex.acquire()
# note - added default value for questionNumber so that this works correctly
# when called from identifier. - Fixes #921
try:
response = self.session.get(
"https://{}/MK/whole/{}/{}".format(self.server, code,
questionNumber),
json={
"user": self.user,
"token": self.token
},
verify=False,
)
response.raise_for_status()
# response should be multipart = [ pageData, f1,f2,f3..]
imagesAsBytes = MultipartDecoder.from_response(response).parts
images = []
i = 0
for iab in imagesAsBytes:
if i == 0:
pageData = json.loads(iab.content)
else:
images.append(BytesIO(
iab.content).getvalue()) # pass back image as bytes
i += 1
except requests.HTTPError as e:
if response.status_code == 401:
raise PlomAuthenticationException() from None
# TODO?
elif response.status_code == 409:
raise PlomTakenException(
"Task taken by another user.") from None
else:
raise PlomSeriousException(
"Some other sort of error {}".format(e)) from None
finally:
self.SRmutex.release()
return [pageData, images]
def MclaimThisTask(self, code):
self.SRmutex.acquire()
try:
response = self.session.patch(
"https://{}/MK/tasks/{}".format(self.server, code),
json={
"user": self.user,
"token": self.token
},
verify=False,
)
response.raise_for_status()
if response.status_code == 204:
raise PlomTakenException("Task taken by another user.")
except requests.HTTPError as e:
if response.status_code == 401:
raise PlomAuthenticationException() from None
else:
raise PlomSeriousException(
"Some other sort of error {}".format(e)) from None
finally:
self.SRmutex.release()
# should be multipart = [tags, integrity_check, image_id_list, image1, image2, ....]
tags = "tagsAndImages[0].text # this is raw text"
imageList = []
i = 0
for img in MultipartDecoder.from_response(response).parts:
if i == 0:
tags = img.text
elif i == 1:
integrity_check = img.text
elif i == 2:
image_id_list = json.loads(img.text)
else:
imageList.append(BytesIO(
img.content).getvalue()) # pass back image as bytes
i += 1
return imageList, image_id_list, tags, integrity_check
def uris(self, database, connection=None):
"""
Get a list of all the URIs in a database.
"""
uris = []
if connection is None:
connection = self.connection
mleval = Eval(connection)
mleval.set_xquery("xquery version '1.0-ml'; cts:uris()")
mleval.set_database(database)
response = mleval.eval()
if 'content-type' in response.headers:
if response.headers['content-type'].startswith("multipart/mixed"):
decoder = MultipartDecoder.from_response(response)
for part in decoder.parts:
uris.append(part.text)
return uris
SERVER = 'localhost'
PORT = 5000
URL = 'http://{}:{}/'.format(SERVER, PORT)
# Files to send
files = {'image.jpg': open('figures/samples/nike.jpg', 'rb')}
# Send file/s
response = requests.post(URL, files=files)
print('[RESPONSE] Status: {}'.format(response.status_code))
# If receiving one file with Flask send_file
# with open('response_file.png', 'wb') as f:
# f.write(response.content)
# If receiving multiple files with MultipartEncoder
decoder = MultipartDecoder(response.content, response.headers['Content-Type'])
for part in decoder.parts:
# Get headers
headers = part.headers
form_data = part.headers.get(b'Content-Disposition').decode('utf-8')
# Get values
name = form_data.split('name="')[1].split('";')[0]
filename = form_data.split('filename="')[1].split('"')[0]
image_path = os.path.join('figures/responses', filename)
print('[RESPONSE] Writing {}'.format(filename))
# Save image
with open(image_path, 'wb') as f:
f.write(part.content)
import sys
import pyaudio
import wave
from requests_toolbelt import MultipartDecoder
filename = sys.argv[1]
boundary = open(filename, 'rb').readline()[2:-2].decode()
decoder = MultipartDecoder(open(filename, 'rb').read(), 'multipart/form-data; boundary={}'.format(boundary), 'latin1')
parts = [part for part in decoder.parts]
print(len(parts))
print(parts[0].content)
print(parts[1].content[:100])
paudio = pyaudio.PyAudio()
waveFile = wave.open('{}.wav'.format(filename), 'wb')
waveFile.setnchannels(1)
waveFile.setsampwidth(paudio.get_sample_size(pyaudio.paInt16))
waveFile.setframerate(16000)
waveFile.writeframes(parts[1].content)
waveFile.close()
def _do_inference(self, model_fn):
"""HTTP endpoint provided by the gateway.
This function should be partially applied with the model_fn argument
before it is added as a Flask route.
Flask functions do not need to take any arguments. They receive the
request data via the module variable flask.request, which is... somehow
always supposed to be accurate within the context of a request-handler.
:param callable model_fn: the callback function to use for inference.
"""
r = flask.request
try:
encoding = r.mimetype_params['charset']
except KeyError:
encoding = 'utf-8'
if not r.content_type.startswith('multipart/related'):
msg = 'invalid content-type {}'.format(r.content_type)
logger.error(msg)
return make_response(msg, 400)
# Decode JSON and DICOMs into BytesIO buffers and pass to model
mp = MultipartDecoder(
content=r.get_data(), content_type=r.content_type,
encoding=encoding
)
input_hash = hashlib.sha256()
for part in mp.parts:
input_hash.update(part.content)
input_digest = input_hash.hexdigest()
logger.debug('received request with hash %s' % input_digest)
test_logger = tagged_logger.TaggedLogger(logger)
test_logger.add_tags({ 'input_hash': input_digest })
request_json_body = json.loads(mp.parts[0].text)
request_binary_dicom_parts = [BytesIO(p.content) for p in mp.parts[1:]]
response_json_body, response_binary_elements = model_fn(
request_json_body, request_binary_dicom_parts, input_digest
)
output_hash = hashlib.sha256()
output_hash.update(json.dumps(response_json_body).encode('utf-8'))
for part in response_binary_elements:
output_hash.update(part)
output_digest = output_hash.hexdigest()
test_logger.add_tags({ 'output_hash': output_digest })
test_logger.debug('request processed')
logger.debug('sending response with hash %s' % output_digest)
# Serialize model response to text
response_body_text_elements = self._serializer(
response_json_body, response_binary_elements
)
# Assemble the list of multipart/related parts
# The json response must be the first part
fields = []
fields.append(
self._make_field_tuple(
'json-body', json.dumps(response_json_body),
content_type='application/json'
)
)
fields.extend(
self._make_field_tuple('elem_{}'.format(i), elem, mimetype)
for i, (mimetype, elem) in enumerate(response_body_text_elements)
)
fields.append(
self._make_field_tuple(
'hashes', input_digest + ':' + output_digest,
content_type='text/plain'
)
)
# Encode using the same boundary and encoding as original
encoder = MultipartEncoder(
fields, encoding=mp.encoding, boundary=mp.boundary
)
# Override the Content-Type header that MultipartEncoder uses
# flask.make_response takes content, response code, and headers
return make_response(
encoder.to_string(), 200,
{'Content-Type': 'multipart/related; boundary={}'.format(mp.boundary)}
)
