def decode_file_upload_multipart_request( request_object_proxy: _RequestObjectProxy, ) -> DecodedFileUploadMultipartRequest: """ Get the decoded parts of a multipart request from a history entry of requests_mock.request_history. This method exists because of limitations with requests_mock: - requests_mock.request_history[n].text is a property method that attempts utf-8 decoding on the body of the request. But we have binary data in our body (we're uploading zip files), so the decode just fails (understandably). To workaround this, we abuse that we have access to the request itself at requests_mock.request_history[n]._request. Then we can access .body on that without decoding. - The object itself from requests_mock.request_history[n] is a private _RequestObjectProxy. This shouldn't be so. It's returned from a public API. Make issue requests for these when able at https://github.com/jamielennox/requests-mock/issues ? """ return DecodedFileUploadMultipartRequest.from_parts( MultipartDecoder( content=request_object_proxy._request.body, content_type=request_object_proxy.headers["Content-Type"], ).parts)
def _get_parts(): content = request.get_data() content_type = request.headers['content-type'] logger.info(f"Received ContentType Header ==> {content_type}") decoder = MultipartDecoder(content, content_type) files = [] for p in decoder.parts: result = dict() result['content'] = p.content content_type = p.headers.get(b'content-type') if content_type is not None: result['content-type'] = content_type.decode("utf-8") content_disposition = p.headers.get(b'content-disposition') if content_disposition is not None: content_disposition = content_disposition.decode("utf-8") m = re.search(r'filename=\"([^\"]*)', content_disposition) result['content-disposition'] = content_disposition if m and m.group(1): result['filename'] = m.group(1) files.append(result) return files
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('AZ Function Post Request for Prediction') try: image_bytes = req.get_body() #logging.info(f'{image_bytes}') content_type = req.headers.get("Content-Type") formdatadecoder = MultipartDecoder(image_bytes, content_type) # assumes there is only one part (our file) being uploaded for part in formdatadecoder.parts: resp_body = predict(part.content) break return func.HttpResponse(status_code=200, headers={"Content-Type": "application/json"}, body=json.dumps(resp_body)) except Exception as e: raise e logging.error(e) return func.HttpResponse( "Either you did not post a compatible image or an unknown error occurred.", status_code=400) return func.HttpResponse()
def parse(self): decoded_content = MultipartDecoder(self.content, self.content_type_header) key_to_content_map = {} for part in decoded_content.parts: cds = part.headers[b'Content-Disposition'] key = ExtractDataParser.get_key_dstring(cds) key_to_content_map[key] = part.content # For Json Response keys: MultiPartKey = ExtractDataParser.get_keys(key_to_content_map) extract_json = key_to_content_map[keys.contentAnalyzerResponse].decode() extract_content_analyzer_res = ExtractContentAnalyzerResponse.from_json(extract_json) extract_output_metadata: ExtractPDFOutputMetadata = extract_content_analyzer_res.outputs # For ElementsInfo structured_output = key_to_content_map[keys.jsonoutput].decode() mt = mimetypes.MimeTypes() extension = mt.guess_extension(extract_output_metadata.indexed_meta_info[keys.jsonoutput].dc_format) structure_output: StructuredData = StructuredData( extension, json.loads(structured_output)) self.ed_zipper.add_structured_data(structure_output) # For Elements Renditions for rendition_key in keys.rendition: rendition_output: ExtractRenditionOutput = self.frame_extract_rendition_output( extract_output_metadata, structure_output, key_to_content_map[rendition_key], rendition_key) self.ed_zipper.add_rendition_data(rendition_output)
def _parse_post_data(self, vector, num, name): """ Parse post data to verify it can be converted into an object by the auditors. This prevents conversion errors for each check and vector combination during the scan. :param vector: vector dictionary :param num: index number :param name: file name :return: True or False """ headers = vector['headers'] post_data = vector['data'] # check data # content-type checked in self._check_content_type() if not post_data: return True # check json if headers['Content-Type'].startswith(HTTP.CONTENT_TYPE.JSON): try: json.loads(post_data) except JSONDecodeError: logger.warning("'postData' is not valid JSON for vector #%d in '%s'. Ignoring.", num, name) return False # check multipart if headers['Content-Type'].startswith(HTTP.CONTENT_TYPE.MULTIPART): try: MultipartDecoder(post_data.encode(), headers['Content-Type']) except (ImproperBodyPartContentException, AttributeError): logger.warning("'postData' is not valid multipart data for vector #%d in '%s'. Ignoring.", num, name) return False # default return True
def do_POST(self): if not self.headers['content-length']: self._make_reasponse( 400, message='You must provide files for transfer.') return file = self.rfile.read(int(self.headers['content-length'])) file_hashes = [] if 'multipart/form-data' in self.headers['content-type']: data = MultipartDecoder(file, content_type=self.headers['content-type']) for part in data.parts: pattern_matches = re.findall( r'.*filename="(.*)".*', part.headers[ 'Content-Disposition'.encode()].decode('utf-8')) if part.headers.get('Content-Type'.encode() ) == 'application/octet-stream'.encode(): file_hashes.append( self._save_file(file=part.content, filename=pattern_matches[0])) else: self._make_reasponse( 400, message= 'Для передачи файлов используйте content-type="multipart/form-data".' ) self._make_reasponse(200, message='\n'.join( map(lambda x: str(x), file_hashes)))
def decode_runs_post_form(event): """Convert the multipart form that comes with a POST to /runs into a dictionary. """ # Decode the form data content_type = event["headers"].get("Content-Type") or event["headers"].get("content-type") decoder = MultipartDecoder(event["body"].encode(), content_type) decoded_body = {} for part in decoder.parts: # parts.headers doesn't seems to support key lookup??? So just iterate # through the values, there are usually two for key, value in part.headers.items(): if key == b"Content-Disposition": content_disposition = cgi.parse_header(value.decode()) name = content_disposition[1]["name"] filename = content_disposition[1].get("filename") break # I guess it's okay to fail if content-disposition doesn't have a name # I think that means the form isn't right if name == "workflow_attachment": decoded_body.setdefault("workflow_attachment", {})[filename] = part.text else: decoded_body[name] = part.text return decoded_body
def parse_multipart(request): content_type = request.headers.get("content-type") if content_type is None or not content_type.startswith( "multipart/related"): utils.error.invalid("Content-type header in multipart upload", None) _, _, boundary = content_type.partition("boundary=") if boundary is None: utils.error.missing( "boundary in content-type header in multipart upload", None) body = extract_media(request) try: decoder = MultipartDecoder(body, content_type) except ImproperBodyPartContentException as e: utils.error.invalid("Multipart body is malformed\n%s" % str(body), None) if len(decoder.parts) != 2: utils.error.invalid("Multipart body is malformed\n%s" % str(body), None) resource = decoder.parts[0].text metadata = json.loads(resource) content_type_key = "content-type".encode("utf-8") headers = decoder.parts[1].headers content_type = {"content-type": headers[content_type_key].decode("utf-8")} media = decoder.parts[1].content return metadata, content_type, media
def test_pushover_success_with_imagebase64(srv, caplog): module = load_module_from_file("mqttwarn/services/pushover.py") image = open("./assets/pushover.png", "rb").read() item = Item( config={}, target="test", addrs=["userkey2", "appkey2"], message="⚽ Notification message ⚽", data={"imagebase64": base64.encodebytes(image)}, ) with caplog.at_level(logging.DEBUG): add_successful_mock_response() outcome = module.plugin(srv, item) # Check response status. assert responses.calls[0].response.status_code == 200 assert responses.calls[0].response.text == '{"status": 1}' # Decode multipart request. request = responses.calls[0].request decoder = MultipartDecoder(request.body, request.headers["Content-Type"]) content_disposition_headers = [] contents = {} for part in decoder.parts: content_disposition_headers.append(part.headers[b"Content-Disposition"]) key = part.headers[b"Content-Disposition"] contents[key] = part.content # Proof request has all body parts. assert content_disposition_headers == [ b'form-data; name="user"', b'form-data; name="token"', b'form-data; name="retry"', b'form-data; name="expire"', b'form-data; name="message"', b'form-data; name="attachment"; filename="image.jpg"', ] # Proof parameter body parts, modulo image content, have correct values. assert list(contents.values())[:-1] == [ b"userkey2", b"appkey2", b"60", b"3600", b"\xe2\x9a\xbd Notification message \xe2\x9a\xbd", ] # Proof image has content. assert len(decoder.parts[-1].content) == 45628 assert outcome is True assert "Sending pushover notification to test" in caplog.text assert "Successfully sent pushover notification" in caplog.text
def multipart_parse(data, content_type, encoding='latin1'): """ parse multipart http response into headers, content tuples :param data: bytes http multipart response body :param content_type: str http response content-type :param encoding: str encoding to use when decoding content :return: """ decoder = MultipartDecoder(data, content_type, encoding) return [body_part_to_headers_and_data(part) for part in decoder.parts]
def parse_response(response) -> typing.Union[bytes, None]: if response.status == http.client.NO_CONTENT: return None if not response.status == http.client.OK: raise HTTPError(response=response) parsed = MultipartDecoder(response.read(), response.headers['content-type'][0].decode()) for part in parsed.parts: if part.headers[b'Content-Type'] == b'application/octet-stream': return part.content
def _get_post_multipart(self): length = int(self.headers['Content-Length']) logging.info("Receiving multipart message") data = self.rfile.read(length) with tempfile.NamedTemporaryFile(delete=False) as f: logging.info("Saving debug data to %s" % f.name) f.write(b"POST /api/mailgun/incoming HTTP/1.1\n") for header in self.headers.keys(): f.write(("%s: %s\n" % (header, self.headers[header])).encode( 'utf-8', 'ignore')) f.write(b"\n") f.write(data) return MultipartDecoder(data, self.headers['Content-Type'])
def _parse_multipart(self, multipart_string, content_type): """ Parse multipart data string and return decoder object. :param multipart_string: multipart data string :param content_type: content-type string :return: Multipart decoder """ # parse try: decoder = MultipartDecoder(multipart_string.encode(), content_type) except ImproperBodyPartContentException: raise InvalidFormatException("Unable to parse multipart form data") except (NonMultipartContentTypeException, AttributeError): raise InvalidFormatException( "Unable to parse multipart content-type") return decoder
def parse_multipart(request): # decode the request decoder = MultipartDecoder(request.raw_body, request.headers['content-type']) body = {} files = {} # loop each part in the multipart upload for p in decoder.parts: data = p.content meta_data = {} # get the content type content_type = p.headers.get(b'content-type', b'').decode() if content_type: meta_data['content-type'] = content_type # parse content disposition content_disposition = p.headers.get(b'content-disposition', b'').decode() if content_disposition: meta = content_disposition.split(';') for m in meta: m = re.sub(r'\s+', '', m) try: # if there is a key value pair here, add to the meta_data for this field k, v = m.split('=') meta_data[k] = v.strip('"') except: pass field_name = meta_data.get('name') # if there is no content-type, just store as a string if not meta_data.get('content-type'): meta_data = data.decode() multi_value_insert(body, field_name, meta_data) if type(meta_data) is dict and data: multi_value_insert(files, field_name, data) return (body, files)
def _do_inference(self, model_fn): """HTTP endpoint provided by the gateway. This function should be partially applied with the model_fn argument before it is added as a Flask route. Flask functions do not need to take any arguments. They receive the request data via the module variable flask.request, which is... somehow always supposed to be accurate within the context of a request-handler. :param callable model_fn: the callback function to use for inference. """ r = flask.request try: encoding = r.mimetype_params['charset'] except KeyError: encoding = 'utf-8' if not r.content_type.startswith('multipart/related'): msg = 'invalid content-type {}'.format(r.content_type) logger.error(msg) return make_response(msg, 400) # Decode JSON and DICOMs into BytesIO buffers and pass to model mp = MultipartDecoder( content=r.get_data(), content_type=r.content_type, encoding=encoding ) input_hash = hashlib.sha256() for part in mp.parts: input_hash.update(part.content) input_digest = input_hash.hexdigest() logger.debug('received request with hash %s' % input_digest) test_logger = tagged_logger.TaggedLogger(logger) test_logger.add_tags({ 'input_hash': input_digest }) request_json_body = json.loads(mp.parts[0].text) request_binary_dicom_parts = [BytesIO(p.content) for p in mp.parts[1:]] response_json_body, response_binary_elements = model_fn( request_json_body, request_binary_dicom_parts, input_digest ) output_hash = hashlib.sha256() output_hash.update(json.dumps(response_json_body).encode('utf-8')) for part in response_binary_elements: output_hash.update(part) output_digest = output_hash.hexdigest() test_logger.add_tags({ 'output_hash': output_digest }) test_logger.debug('request processed') logger.debug('sending response with hash %s' % output_digest) # Serialize model response to text response_body_text_elements = self._serializer( response_json_body, response_binary_elements ) # Assemble the list of multipart/related parts # The json response must be the first part fields = [] fields.append( self._make_field_tuple( 'json-body', json.dumps(response_json_body), content_type='application/json' ) ) fields.extend( self._make_field_tuple('elem_{}'.format(i), elem, mimetype) for i, (mimetype, elem) in enumerate(response_body_text_elements) ) fields.append( self._make_field_tuple( 'hashes', input_digest + ':' + output_digest, content_type='text/plain' ) ) # Encode using the same boundary and encoding as original encoder = MultipartEncoder( fields, encoding=mp.encoding, boundary=mp.boundary ) # Override the Content-Type header that MultipartEncoder uses # flask.make_response takes content, response code, and headers return make_response( encoder.to_string(), 200, {'Content-Type': 'multipart/related; boundary={}'.format(mp.boundary)} )
def __init__(self, response): parsed_response = MultipartDecoder( response.read(), response.headers['content-type'][0].decode()) self.parts = parsed_response.parts
from requests_toolbelt import MultipartDecoder from PIL import Image from io import BytesIO with open("tmp/2018-12-03-14-32-53", "r") as f: metadata = f.read() bdry = metadata.split("\n")[0][2:] decoder = MultipartDecoder(metadata, 'multipart/form-data;boundary={}'.format(bdry)) image = Image.open(BytesIO(decoder.parts[1].content)) image.save('test.jpg')
import sys import pyaudio import wave from requests_toolbelt import MultipartDecoder filename = sys.argv[1] boundary = open(filename, 'rb').readline()[2:-2].decode() decoder = MultipartDecoder(open(filename, 'rb').read(), 'multipart/form-data; boundary={}'.format(boundary), 'latin1') parts = [part for part in decoder.parts] print(len(parts)) print(parts[0].content) print(parts[1].content[:100]) paudio = pyaudio.PyAudio() waveFile = wave.open('{}.wav'.format(filename), 'wb') waveFile.setnchannels(1) waveFile.setsampwidth(paudio.get_sample_size(pyaudio.paInt16)) waveFile.setframerate(16000) waveFile.writeframes(parts[1].content) waveFile.close()
def do_POST(self): database = getDatabase() session = { 'username': None, 'apiKey': None, 'isAuthorised': None, 'binaryFile': None, 'originalFilename': None } if 'API-Key' in self.headers: cursor = database.execute( 'SELECT username,apiKey FROM `users` WHERE `apiKey` = ?', (self.headers['API-Key'], )) row = cursor.fetchone() if row: username, apiKey = row session['username'] = username session['apiKey'] = apiKey session['isAuthorised'] = True else: self.send_error(HTTPStatus.FORBIDDEN, message='You must provide a valid API key.') return else: self.send_error(HTTPStatus.FORBIDDEN, message='You must provide an API key') return if int(self.headers['Content-Length']) > int( os.getenv('MAX_UPLOAD_BYTES', 8388608)): self.send_error( HTTPStatus.NOT_ACCEPTABLE, message='You have exceeded filelimit size, kgg over') return multipart_decoder = MultipartDecoder( self.rfile.read(int(self.headers['Content-Length'])), self.headers['Content-Type']) for part in multipart_decoder.parts: for index, value in part.headers.items(): index, value = cgi.parse_header(value.decode('utf-8')) if 'name' in value: if value['name'] == 'upload': session['binaryFile'] = part.content if 'filename' in value: session['originalFilename'] = value['filename'] if session['isAuthorised'] and session['binaryFile']: if not mimetypes.inited: mimetypes.init() mimetype, encoding = mimetypes.guess_type( session['originalFilename']) if mimetypes.guess_type( session['originalFilename'])[0] else ( 'application/octet-stream', None, ) media_body = apiclient.http.MediaIoBaseUpload(io.BytesIO( session['binaryFile']), mimetype=mimetype) body = { 'name': base64.b64encode( bytearray( '{}.{}'.format(datetime.datetime.now(), session['originalFilename']), 'utf-8')).decode('utf-8'), 'parents': [os.getenv('ROOT_FOLDER')], 'originalFilename': session['originalFilename'], 'createdTime': datetime.datetime.utcnow().isoformat() + 'Z' } new_file = getGoogle().files().create( body=body, media_body=media_body, fields= 'id,name,originalFilename,md5Checksum,createdTime,mimeType,size' ).execute() cursor = database.execute( 'INSERT INTO `uploads` (googleId, name, originalFilename, mimeType, md5Checksum, createdTime, size) VALUES (?,?,?,?,?,?,?)', ( new_file['id'], new_file['name'], new_file['originalFilename'], new_file['mimeType'], new_file['md5Checksum'], new_file['createdTime'], new_file['size'], )) database.commit() jsons = json.dumps( {'url': '{}/{}'.format(os.getenv('URL'), new_file['id'])}) self.send_response(HTTPStatus.OK) self.send_header('Content-type', 'text/plain') self.send_header("Content-Length", len(jsons)) self.end_headers() self.wfile.write(bytearray(jsons, 'utf-8')) else: self.send_error(HTTPStatus.FORBIDDEN) return
SERVER = 'localhost' PORT = 5000 URL = 'http://{}:{}/'.format(SERVER, PORT) # Files to send files = {'image.jpg': open('figures/samples/nike.jpg', 'rb')} # Send file/s response = requests.post(URL, files=files) print('[RESPONSE] Status: {}'.format(response.status_code)) # If receiving one file with Flask send_file # with open('response_file.png', 'wb') as f: # f.write(response.content) # If receiving multiple files with MultipartEncoder decoder = MultipartDecoder(response.content, response.headers['Content-Type']) for part in decoder.parts: # Get headers headers = part.headers form_data = part.headers.get(b'Content-Disposition').decode('utf-8') # Get values name = form_data.split('name="')[1].split('";')[0] filename = form_data.split('filename="')[1].split('"')[0] image_path = os.path.join('figures/responses', filename) print('[RESPONSE] Writing {}'.format(filename)) # Save image with open(image_path, 'wb') as f: f.write(part.content)
def parse_multipart(body, content_type, boundary=b'--boundary'): # remove the frame header as that prevents parsing the multipart request # (generated in frame serialization) body = boundary.join(body.split(boundary)[1:]) return MultipartDecoder(body, content_type)