def test_iterlines_maxbuf(self): data, limit = 'abcdefgh\nijklmnop\r\nq', 9 result = [(to_bytes('abcdefgh'),to_bytes('\n')),(to_bytes('ijklmnop'),to_bytes('')),(to_bytes(''),to_bytes('\r\n')),(to_bytes('q'),to_bytes(''))] i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo', buffer_size=limit)._lineiter() self.assertEqual(list(i), result) data, limit = ('X'*3*1024)+'x\n', 1024 result = [(to_bytes('X'*1024),to_bytes('')),(to_bytes('X'*1024),to_bytes('')),(to_bytes('X'*1024),to_bytes('')),(to_bytes('x'),to_bytes('\n'))] i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo', buffer_size=limit)._lineiter() self.assertEqual(list(i), result)
def test_iterlines_limit(self): data, limit = 'abc\ndef\r\nghi', 10 result = [(to_bytes('abc'),to_bytes('\n')),(to_bytes('def'),to_bytes('\r\n')),(to_bytes('g'),to_bytes(''))] i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo', limit)._lineiter() self.assertEqual(list(i), result) data, limit = 'abc\ndef\r\nghi', 8 result = [(to_bytes('abc'),to_bytes('\n')),(to_bytes('def'),to_bytes('\r'))] i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo', limit)._lineiter() self.assertEqual(list(i), result)
def decode_multiform_data(data): s_pat = data.split(b"\r")[0][2:] parts = multipart.MultipartParser(BytesIO(multipart.to_bytes(data)), s_pat).parts() d_d = {} for part_t in parts: if part_t.content_type != '': d_d[part_t.name] = { 'isfile': True, 'filename': part_t.filename, 'name': part_t.name, 'file': part_t.file, 'size': part_t.size, } else: try: value = tornado.escape.json_decode(part_t.value) except: value = part_t.value # endtry d_d[part_t.name] = { 'isfile': False, 'name': part_t.name, 'value': value, } # endif # endfor return d_d
def test_big_file(self): ''' If the size of an uploaded part exceeds memfile_limit, it is written to disk. ''' test_file = 'abc' * 1024 boundary = '---------------------------186454651713519341951581030105' request = BytesIO( to_bytes('\r\n').join( map(to_bytes, [ '--' + boundary, 'Content-Disposition: form-data; name="file1"; filename="random.png"', 'Content-Type: image/png', '', test_file, '--' + boundary, 'Content-Disposition: form-data; name="file2"; filename="random.png"', 'Content-Type: image/png', '', test_file + 'a', '--' + boundary, 'Content-Disposition: form-data; name="file3"; filename="random.png"', 'Content-Type: image/png', '', test_file * 2, '--' + boundary + '--', '' ]))) p = mp.MultipartParser(request, boundary, memfile_limit=len(test_file)) self.assertEqual(p.get('file1').file.read(), to_bytes(test_file)) self.assertTrue(p.get('file1').is_buffered()) self.assertEqual(p.get('file2').file.read(), to_bytes(test_file + 'a')) self.assertFalse(p.get('file2').is_buffered()) self.assertEqual(p.get('file3').file.read(), to_bytes(test_file * 2)) self.assertFalse(p.get('file3').is_buffered())
def test_save_as(self): ''' save_as stores data in a file keeping the file position. ''' def tmp_file_name(): # create a temporary file name (on Python 2.6+ NamedTemporaryFile # with delete=False could be used) fd, fname = tempfile.mkstemp() f = os.fdopen(fd) f.close() return fname test_file = 'abc' * 1024 boundary = '---------------------------186454651713519341951581030105' request = BytesIO( to_bytes('\r\n').join( map(to_bytes, [ '--' + boundary, 'Content-Disposition: form-data; name="file1"; filename="random.png"', 'Content-Type: image/png', '', test_file, '--' + boundary + '--', '' ]))) p = mp.MultipartParser(request, boundary) self.assertEqual( p.get('file1').file.read(1024), to_bytes(test_file)[:1024]) tfn = tmp_file_name() p.get('file1').save_as(tfn) tf = open(tfn, 'rb') self.assertEqual(tf.read(), to_bytes(test_file)) tf.close() self.assertEqual( p.get('file1').file.read(), to_bytes(test_file)[1024:])
def post_file_callback(request): s = request.body.split(b"\r")[0][2:] p = list( multipart.MultipartParser(BytesIO(multipart.tob(request.body)), s))[0] assert "filename" in p.options return 200, JSON_HEADER, json.dumps({"test": "test"})
def test_iterlines(self): data = 'abc\ndef\r\nghi' result = [(to_bytes('abc'), to_bytes('\n')), (to_bytes('def'), to_bytes('\r\n')), (to_bytes('ghi'), to_bytes(''))] i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo')._lineiter() self.assertEqual(list(i), result)
def do_POST(self): host = self.client_address[0] try: content_length = int(self.headers['Content-Length']) body = self.rfile.read(content_length) stream = BytesIO(body) boundary = stream.readline() boundary = boundary.strip(b"\r\n")[2:] stream.seek(0) parser = multipart.MultipartParser(stream, boundary) for part in parser: # Try not to allow uploads to things outside of upload_dir. # WARNING: probably not secure localpath = os.path.join( args.upload_dir, os.path.basename(os.path.realpath(part.name))) print("Submission {} uploading to {}".format( part.name, localpath)) with open(localpath, 'wb') as f: shutil.copyfileobj(part.file, f) except Exception as e: print(e) self._reply(str(e).encode()) else: self._reply(b"Upload successful!\n")
def bytes_to_fragments(b): ret = [] bio = io.BytesIO(b) parser = multipart.MultipartParser(bio, b"MIME-BOUNDARY", charset=None) for part in parser: ret.append(Fragment(part.name, part.raw)) return ret
def read_form_data(self): content_length = int(self.headers['Content-Length']) post_data = self.rfile.read(content_length).decode("utf8") parser = multipart.MultipartParser( BytesIO(multipart.to_bytes(post_data)), post_data.split("\r")[0][2:]) parts = parser.parts() return {part.name: part.value for part in parts}
def test_file_seek(self): ''' The file object should be readable withoud a seek(0). ''' test_file = 'abc'*1024 boundary = '---------------------------186454651713519341951581030105' request = BytesIO(to_bytes('\r\n').join(map(to_bytes,[ '--' + boundary, 'Content-Disposition: form-data; name="file1"; filename="random.png"', 'Content-Type: image/png', '', test_file, '--' + boundary + '--','']))) p = mp.MultipartParser(request, boundary) self.assertEqual(p.get('file1').file.read(), to_bytes(test_file)) self.assertEqual(p.get('file1').value, test_file)
def test_unicode_value(self): ''' The .value property always returns unicode ''' test_file = 'abc'*1024 boundary = '---------------------------186454651713519341951581030105' request = BytesIO(to_bytes('\r\n').join(map(to_bytes,[ '--' + boundary, 'Content-Disposition: form-data; name="file1"; filename="random.png"', 'Content-Type: image/png', '', test_file, '--' + boundary + '--','']))) p = mp.MultipartParser(request, boundary) self.assertEqual(p.get('file1').file.read(), to_bytes(test_file)) self.assertEqual(p.get('file1').value, test_file) self.assertTrue(hasattr(p.get('file1').value, 'encode'))
def do_upload(self): # Do some browsers /really/ use multipart ? maybe Opera ? try: #self.log_message("Started file transfer") content_length = int(self.headers['Content-Length']) body = self.rfile.read(content_length) print("Downloading {} bytes...".format(content_length)) #print("Content Type: {}".format(self.headers.get_content_type())) #print("Body: {}".format(body)) stream = BytesIO(body) boundary = stream.readline() boundary = boundary.strip(b"\r\n")[2:] #print("Boundary: {}".format(boundary)) stream.seek(0) parser = multipart.MultipartParser(stream, boundary) #print("{}".format(parser)) #print("Data:\n{}".format(parser.get('data').file.read())) for part in parser: host = self.get_host().replace(":", "_") #print("Joining {}+{}+{}".format(SETTINGS['exfil_dir'], host , part.name)) localpath = os.path.join(SETTINGS['exfil_dir'], host, part.name.lstrip("/")) #localpath = os.path.join(localpath , part.name) #.encode('utf-8') #print("Initial path: {}".format(localpath)) root, ext = os.path.splitext(localpath) i = 1 try: os.makedirs(os.path.dirname(localpath), 755) except Exception as e: pass # race condition, but hey... while (os.path.exists(localpath)): localpath = "%s-%d%s" % (root, i, ext) i = i + 1 #print("Writing to: {}".format(localpath)) fout = open(localpath, 'wb') shutil.copyfileobj(part.file, fout) fout.close() os.chmod(localpath, 755) #self.log_message("Received: %s", os.path.basename(localpath)) #self.send_html(self.html("success")) print("Downloaded: {}".format(localpath)) except Exception as e: #self.log_message(repr(e)) print("Error downloading: {}".format(e)) traceback.print_exc()
async def parse_formdata(body_type, request: aiohttp.web.BaseRequest) -> dict: indata = {} for key, val in urllib.parse.parse_qsl(request.query_string): indata[key] = val # PUT/POST form data? if request.method in ["PUT", "POST"]: if request.can_read_body: try: if (request.content_length and request.content_length > PYPONY_MAX_PAYLOAD): raise ValueError( "Form data payload too large, max 256kb allowed") body = await request.text() if body_type == "json": try: js = json.loads(body) assert isinstance( js, dict ) # json data MUST be an dictionary object, {...} indata.update(js) except ValueError: raise ValueError("Erroneous payload received") elif body_type == "form": if (request.headers.get("content-type", "").lower() == "application/x-www-form-urlencoded"): try: for key, val in urllib.parse.parse_qsl(body): indata[key] = val except ValueError: raise ValueError("Erroneous payload received") # If multipart, turn our body into a BytesIO object and use multipart on it elif ("multipart/form-data" in request.headers.get("content-type", "").lower()): fh = request.headers.get("content-type", "") fb = fh.find("boundary=") if fb > 0: boundary = fh[fb + 9:] if boundary: try: for part in multipart.MultipartParser( io.BytesIO(body.encode("utf-8")), boundary, len(body), ): indata[part.name] = part.value except ValueError: raise ValueError( "Erroneous payload received") finally: pass return indata
def upload_file(): if request.method == 'POST': data = request.data #s = str(data).split("\r")[0][16:] s = data[2:42] print("s is ", s) p = mp.MultipartParser(BytesIO(tob(data)),s) newFile = open("/tmp/screenshot.png", "wb") #newFile.write(data) newFile.write((p.parts()[0].value.encode("latin-1"))) #print("Data size: {}".format(len(request.data))) return "OK"
def resolve_multipart(data) -> dict: boundary = data.decode().split("\r")[0][2:] multipart_parser = mp.MultipartParser(BytesIO(data), boundary) blob = multipart_parser.parts()[0].value temp_file_name = tempfile.NamedTemporaryFile(delete=False).name f = open(temp_file_name, "wb") f.write(blob.encode("latin-1")) f.close() parameters = multipart_parser.parts()[1].value parameters_json = yaml.load(parameters, Loader=yaml.SafeLoader) parameters_json['workflow_yaml'] = temp_file_name return parameters_json
def test_get_all(self): ''' Test the get() and get_all() methods. ''' boundary = '---------------------------186454651713519341951581030105' request = BytesIO(to_bytes('\r\n').join(map(to_bytes,[ '--' + boundary, 'Content-Disposition: form-data; name="file1"; filename="random.png"', 'Content-Type: image/png', '', 'abc'*1024, '--' + boundary, 'Content-Disposition: form-data; name="file1"; filename="random.png"', 'Content-Type: image/png', '', 'def'*1024, '--' + boundary + '--','']))) p = mp.MultipartParser(request, boundary) self.assertEqual(p.get('file1').file.read(), to_bytes('abc'*1024)) self.assertEqual(p.get('file2'), None) self.assertEqual(len(p.get_all('file1')), 2) self.assertEqual(p.get_all('file1')[1].file.read(), to_bytes('def'*1024)) self.assertEqual(p.get_all('file1'), p.parts())
def parse_body(body, ctype, projname, dm_type): body_n = None if "multipart/form-data" in ctype: # body = str(body) # COMMENT: old code line body = body.encode("utf-8") # COMMENT: code line that does not run into encoding error # COMMENT: parse multipart/form-data body body_n = ParseTree(projname, MULTIPART, body) # s_obj = StringIO(body.encode("utf-8")) # COMMENT: old code line s_obj = StringIO(body) # new code line reflecting previous changes boundary = ctype.split("; boundary=")[1] mp = multipart.MultipartParser(s_obj, boundary) pos = 0 for part in mp: k = part.options.get("name") v = part.value kv_n = PTNonTerminalNode(projname, MULTIPART, "multipart-pair", pos) k_n = PTTerminalNode(projname, dm_type, k, "multipart-name", 0) v_n = PTTerminalNode(projname, dm_type, v, "multipart-value", 1) kv_n.HasChild.add(k_n) kv_n.HasChild.add(v_n) body_n.HasChild.add(kv_n) pos += 1 elif "application/x-www-form-urlencoded" in ctype: body = str(body) body_n = ParseTree(projname, FORMURLENC, body) pos = 0 for k, vs in parse_qs(body).iteritems(): for v in vs: pair_n = PTNonTerminalNode(projname, URL, "form-urlenc-pair", pos) pair_n.HasChild.add(PTTerminalNode(projname, FORMURLENC, k, "form-urlenc-param-name", 0)) pair_n.HasChild.add(PTTerminalNode(projname, FORMURLENC, v, "form-urlenc-param-value", 1)) body_n.HasChild.add(pair_n) pos += 1 elif "json" in ctype: body = str(body) body_n = ParseTree(projname, JSON, body) cnt = visit_json(projname, json.loads(body)) body_n.HasChild.add(cnt) else: body_n = ParseTree(projname, ctype, "{} file, we ignore this content".format(ctype)) s_n = PTTerminalNode(projname, dm_type, "{} file, we ignore this content".format(ctype), "plaintext-body", 0) body_n.HasChild.add(s_n) return body_n
def do_POST(self): try: content_length = int(self.headers['Content-Length']) body = self.rfile.read(content_length) stream = BytesIO(body) boundary = stream.readline() boundary = boundary.strip(b"\r\n")[2:] stream.seek(0) parser = multipart.MultipartParser(stream, boundary) for part in parser: res = part.file.read().decode() if res: print(res) except Exception as e: print(e)
def nbformathandler(event, context): headers = event['headers']; content_type = headers['content-type'] boundary = content_type.split('=')[1] httpbody = mp.MultipartParser(BytesIO(str.encode(event['body'])), boundary) data = httpbody.get('data').file.read() in_memory_source = BytesIO(data) res = execute_notebook(in_memory_source) response = { "statusCode": 200, "body": res } return response
def test_multiline_header(self): ''' HTTP allows headers to be multiline. ''' test_file = to_bytes('abc'*1024) test_text = u'Test text\n with\r\n ümläuts!' boundary = '---------------------------186454651713519341951581030105' request = BytesIO(to_bytes('\r\n').join(map(to_bytes,[ '--' + boundary, 'Content-Disposition: form-data;', '\tname="file1"; filename="random.png"', 'Content-Type: image/png', '', test_file, '--' + boundary, 'Content-Disposition: form-data;', ' name="text"', '', test_text, '--' + boundary + '--','']))) p = mp.MultipartParser(request, boundary, charset='utf8') self.assertEqual(p.get('file1').file.read(), test_file) self.assertEqual(p.get('file1').filename, 'random.png') self.assertEqual(p.get('text').value, test_text)
def do_POST(self): if self.path.lower().strip('/') != str(guid): self.send_error(HTTPStatus.NOT_FOUND, 'File not found') return None # Receive and store file content_length = int(self.headers['Content-Length']) _, options = cgi.parse_header(self.headers['Content-Type']) boundary = options['boundary'] data = io.BytesIO(self.rfile.read(content_length)) parser = multipart.MultipartParser(data, boundary) selector_data = None for part in parser.parts(): if part.name == 'fileselector': selector_data = part.raw filename = part.filename break else: print('Error: Data not found') self.send_error(HTTPStatus.BAD_REQUEST, 'Improperly formed request') new_file_path = file_path / filename if new_file_path.exists(): self.send_response(HTTPStatus.OK) message = FAIL_FORM_HTML.format('File already exists').encode() else: with open(file_path / filename, 'wb') as f: f.write(selector_data) print(f'Received {filename}') self.send_response(HTTPStatus.CREATED) message = SUCCESS_FORM_HTML.encode() self.send_header("Content-type", 'text/html') self.send_header("Content-Length", str(len(message))) self.end_headers() self.wfile.write(message) self.wfile.flush()
def _multipart_form_parser(request): _, options = cgi.parse_header(request['content_type']) boundary = options['boundary'] parser = multipart.MultipartParser(request.body, boundary) multipart_params = m().evolver() form_params = m().evolver() for part in parser.parts(): if part.disposition == 'form-data' and not part.content_type: form_params[part.name] = part.value multipart_params[part.name] = m( content_type=part.content_type or None, content_length=part.size, headers=pmap(part.headerlist), name=part.name, filename=part.filename, character_encoding=part.charset, body=part.file) request = request.transform( ['form_params'], lambda params: params.update(form_params.persistent())) return request.set('multipart_params', multipart_params.persistent())
def decode_multipart(o, content_type, **kwargs): har = [] try: if isinstance(o, six.string_types): import multipart content_type, options = multipart.parse_options_header( content_type) assert content_type == 'multipart/form-data' stream = six.BytesIO(o) boundary = six.binary_type(options.get('boundary')) assert boundary for part in multipart.MultipartParser(stream, boundary, len(o), **kwargs): if part.filename or not part.is_buffered(): param = {'name': part.name, 'value': part.value, 'filename': part.filename} else: # TODO: Big form-fields are in the files dict. really? param = {'name': part.name, 'value': part.value} har.append(param) except Exception as err: print(repr(err)) return har
def test_line_parser(self): for line in ('foo', ''): for ending in ('\n', '\r', '\r\n'): i = mp.MultipartParser(BytesIO(to_bytes(line + ending)), 'foo') i = next(i._lineiter()) self.assertEqual(i, (to_bytes(line), to_bytes(ending)))
async def parse(self) -> FormData: # Parse the Content-Type header to get the multipart boundary. content_type, params = parse_options_header(self.headers["Content-Type"]) boundary = params.get(b"boundary") # Callbacks dictionary. callbacks = { "on_part_begin": self.on_part_begin, "on_part_data": self.on_part_data, "on_part_end": self.on_part_end, "on_header_field": self.on_header_field, "on_header_value": self.on_header_value, "on_header_end": self.on_header_end, "on_headers_finished": self.on_headers_finished, "on_end": self.on_end, } # Create the parser. parser = multipart.MultipartParser(boundary, callbacks) header_field = b"" header_value = b"" raw_headers = [] # type: typing.List[typing.Tuple[bytes, bytes]] field_name = "" data = b"" file = None # type: typing.Optional[UploadFile] items = ( [] ) # type: typing.List[typing.Tuple[str, typing.Union[str, UploadFile]]] # Feed the parser with data from the request. async for chunk in self.stream: parser.write(chunk) messages = list(self.messages) self.messages.clear() for message_type, message_bytes in messages: if message_type == MultiPartMessage.PART_BEGIN: raw_headers = [] data = b"" elif message_type == MultiPartMessage.HEADER_FIELD: header_field += message_bytes elif message_type == MultiPartMessage.HEADER_VALUE: header_value += message_bytes elif message_type == MultiPartMessage.HEADER_END: raw_headers.append((header_field.lower(), header_value)) header_field = b"" header_value = b"" elif message_type == MultiPartMessage.HEADERS_FINISHED: headers = Headers(raw=raw_headers) content_disposition = headers.get("Content-Disposition") content_type = headers.get("Content-Type", "") disposition, options = parse_options_header(content_disposition) field_name = options[b"name"].decode("latin-1") if b"filename" in options: filename = options[b"filename"].decode("latin-1") file = UploadFile(filename=filename, content_type=content_type) else: file = None elif message_type == MultiPartMessage.PART_DATA: if file is None: data += message_bytes else: await file.write(message_bytes) elif message_type == MultiPartMessage.PART_END: if file is None: items.append((field_name, data.decode("latin-1"))) else: await file.seek(0) items.append((field_name, file)) elif message_type == MultiPartMessage.END: pass parser.finalize() return FormData(items)
def opentc_REQMOD(self): self.multipart_data = None self.last_form_field = None self.big_chunk = b'' self.content_analysis_results = dict() try: response = self.server.opentc["client"].command("PING\n") response = json.loads(response.decode('utf-8')) self.logger.debug("REQMOD Ping response: {}".format(response)) except Exception as err: self.logger.error(traceback.format_exc()) def on_part_begin(): self.multipart_data = dict() self.multipart_data[b'Content'] = b'' self.logger.debug("on_part_begin") def on_part_data(data, start, end): self.multipart_data[b'Content'] += data[start:end] self.logger.debug("on_part_data") def on_part_end(): self.logger.debug("on_part_end") for key in self.multipart_data: if key == b'Content': mime_type = magic.from_buffer( self.multipart_data[b'Content'], mime=True) self.logger.debug( "Content mime_type: {}".format(mime_type)) if b'Content-Type' in self.multipart_data: # content_type = [ct.strip() for ct in self.multipart_data[b'Content-Type'].split(b';')] content_type = [mime_type] content_disposition = {'name': '', 'filename': ''} for x in self.multipart_data[ b'Content-Disposition'].split(b';'): if b'=' in x: key, value = x.split(b'=') key = key.decode("utf-8").strip(" \"") value = value.decode("utf-8").strip(" \"") content_disposition[key] = value print(content_disposition) result = self.content_analyse( converter=self.server.opentc["config"] ["converter"], content_disposition=content_disposition, content_type=content_type, content=self.multipart_data[b'Content'], content_min_length=self.server.opentc["config"] ["content_min_length"], client=self.server.opentc["client"]) name = self.multipart_data[ b'Content-Disposition'].split(b';')[1].split( b'=')[1] self.content_analysis_results[name.decode( "utf-8").replace('"', '')] = result else: self.logger.debug("{}: {}".format( key, self.multipart_data[key])) return def on_header_field(data, start, end): self.last_form_field = data[start:end] self.logger.debug("on_header_field") def on_header_value(data, start, end): self.multipart_data[self.last_form_field] = data[start:end] self.logger.debug("on_header_value") def on_end(): self.logger.debug("on_end") self.set_icap_response(200) # self.set_enc_request(b' '.join(self.enc_req)) for h in self.enc_req_headers: for v in self.enc_req_headers[h]: self.set_enc_header(h, v) # Copy the request body (in case of a POST for example) if not self.has_body: self.set_enc_request(b' '.join(self.enc_req)) self.send_headers(False) return if self.preview: prevbuf = b'' while True: chunk = self.read_chunk() if chunk == b'': break prevbuf += chunk if self.ieof: self.send_headers(True) if len(prevbuf) > 0: self.write_chunk(prevbuf) self.write_chunk(b'') return self.cont() self.set_enc_request(b' '.join(self.enc_req)) self.send_headers(True) if len(prevbuf) > 0: self.write_chunk(prevbuf) while True: chunk = self.read_chunk() self.write_chunk(chunk) if chunk == b'': break else: # Parse the Content-Type header to get the multipart boundary. content_type, params = parse_options_header( self.enc_req_headers[b'content-type'][0]) boundary = params.get(b'boundary') parser = None if boundary is not None: # Callbacks dictionary. callbacks = { 'on_part_begin': on_part_begin, 'on_part_data': on_part_data, 'on_part_end': on_part_end, 'on_header_field': on_header_field, 'on_header_value': on_header_value, 'on_end': on_end } parser = multipart.MultipartParser(boundary, callbacks) while True: chunk = self.read_chunk() if chunk == b'': break self.big_chunk += chunk if boundary is not None: size = len(self.big_chunk) start = 0 while size > 0: end = min(size, 1024 * 1024) parser.write(self.big_chunk[start:end]) size -= end start = end else: result = self.content_analyse( converter=self.server.opentc["config"]["converter"], content_type=content_type, content=self.big_chunk, content_min_length=self.server.opentc["config"] ["content_min_length"], client=self.server.opentc["client"]) name = "text" self.content_analysis_results[name] = result is_allowed = True for result in self.content_analysis_results: if self.content_analysis_results[result] is None: continue for classifier in self.server.opentc["config"][ "classifier_status"]: if self.server.opentc["config"]["classifier_status"][ classifier] is False: continue for restricted_class in self.server.opentc["config"][ "restricted_classes"]: self.logger.debug( "{}: result:{}, classifier:{}".format( restricted_class, result, classifier)) if restricted_class in self.content_analysis_results[ result][classifier]: is_allowed = False break else: is_allowed = True if is_allowed is True: break if is_allowed is False: break if is_allowed: self.set_enc_request(b' '.join(self.enc_req)) self.send_headers(True) self.write_chunk(self.big_chunk) else: content = json.dumps(self.content_analysis_results) content = "result={}".format(content).encode("utf-8") enc_req = self.enc_req[:] enc_req[0] = self.server.opentc["config"][ "replacement_http_method"].encode("utf-8") enc_req[1] = self.server.opentc["config"][ "replacement_url"].encode("utf-8") self.set_enc_request(b' '.join(enc_req)) self.enc_headers[b"content-type"] = [ b"application/x-www-form-urlencoded" ] self.enc_headers[b"content-length"] = [ str(len(content)).encode("utf-8") ] self.send_headers(True) self.write_chunk(content)
async def parse(self) -> FormData: # Parse the Content-Type header to get the multipart boundary. content_type, params = parse_options_header( self.headers["Content-Type"]) charset = params.get(b"charset", "utf-8") if type(charset) == bytes: charset = charset.decode("latin-1") boundary = params[b"boundary"] # Callbacks dictionary. callbacks = { "on_part_begin": self.on_part_begin, "on_part_data": self.on_part_data, "on_part_end": self.on_part_end, "on_header_field": self.on_header_field, "on_header_value": self.on_header_value, "on_header_end": self.on_header_end, "on_headers_finished": self.on_headers_finished, "on_end": self.on_end, } # Create the parser. parser = multipart.MultipartParser(boundary, callbacks) header_field = b"" header_value = b"" content_disposition = None content_type = b"" field_name = "" data = b"" file: typing.Optional[UploadFile] = None items: typing.List[typing.Tuple[str, typing.Union[str, UploadFile]]] = [] item_headers: typing.List[typing.Tuple[bytes, bytes]] = [] # Feed the parser with data from the request. async for chunk in self.stream: parser.write(chunk) messages = list(self.messages) self.messages.clear() for message_type, message_bytes in messages: if message_type == MultiPartMessage.PART_BEGIN: content_disposition = None content_type = b"" data = b"" item_headers = [] elif message_type == MultiPartMessage.HEADER_FIELD: header_field += message_bytes elif message_type == MultiPartMessage.HEADER_VALUE: header_value += message_bytes elif message_type == MultiPartMessage.HEADER_END: field = header_field.lower() if field == b"content-disposition": content_disposition = header_value elif field == b"content-type": content_type = header_value item_headers.append((field, header_value)) header_field = b"" header_value = b"" elif message_type == MultiPartMessage.HEADERS_FINISHED: disposition, options = parse_options_header( content_disposition) field_name = _user_safe_decode(options[b"name"], charset) if b"filename" in options: filename = _user_safe_decode(options[b"filename"], charset) file = UploadFile( filename=filename, content_type=content_type.decode("latin-1"), headers=Headers(raw=item_headers), ) else: file = None elif message_type == MultiPartMessage.PART_DATA: if file is None: data += message_bytes else: await file.write(message_bytes) elif message_type == MultiPartMessage.PART_END: if file is None: items.append( (field_name, _user_safe_decode(data, charset))) else: await file.seek(0) items.append((field_name, file)) parser.finalize() return FormData(items)
def _upload_files(self, req, module, user_id, resp): # Soubory bez specifikace delky neberem. if not req.content_length: resp.status = falcon.HTTP_411 req.context['result'] = { 'result': 'error', 'error': 'Nelze nahrát neukončený stream.' } return # Prilis velke soubory neberem. if req.content_length > util.config.MAX_UPLOAD_FILE_SIZE: resp.status = falcon.HTTP_413 req.context['result'] = { 'result': 'error', 'error': 'Maximální velikost dávky je 20 MB.' } return # Pokud uz existuji odevzdane soubory, nevytvarime nove # evaluation, pouze pripojujeme k j*z existujicimu try: existing = util.module.existing_evaluation(module.id, user_id) if len(existing) > 0: evaluation = session.query(model.Evaluation).get(existing[0]) evaluation.time = datetime.datetime.utcnow() report = evaluation.full_report else: report = (str(datetime.datetime.now()) + ' : === Uploading files for module id \'%s\' for ' 'task id \'%s\' ===\n' % (module.id, module.task)) evaluation = model.Evaluation(user=user_id, module=module.id, ok=True) session.add(evaluation) session.commit() # Lze uploadovat jen omezeny pocet souboru. file_cnt = session.query(model.SubmittedFile).\ filter(model.SubmittedFile.evaluation == evaluation.id).count() if file_cnt > util.config.MAX_UPLOAD_FILE_COUNT: resp.status = falcon.HTTP_400 req.context['result'] = { 'result': 'error', 'error': 'K řešení lze nahrát nejvýše 20 souborů.' } return except SQLAlchemyError: session.rollback() raise dir = util.module.submission_dir(module.id, user_id) try: os.makedirs(dir) except OSError: pass if not os.path.isdir(dir): resp.status = falcon.HTTP_400 req.context['result'] = { 'result': 'error', 'error': 'Chyba 42, kontaktuj orga.' } return files = multipart.MultiDict() content_type, options = multipart.parse_options_header( req.content_type) boundary = options.get('boundary', '') if not boundary: raise multipart.MultipartError( "No boundary for multipart/form-data.") for part in multipart.MultipartParser(req.stream, boundary, req.content_length, 2**30, 2**20, 2**18, 2**16, 'utf-8'): path = '%s/%s' % (dir, part.filename) part.save_as(path) mime = magic.Magic(mime=True).from_file(path) report += (str(datetime.datetime.now()) + ' : [y] uploaded file: \'%s\' (mime: %s) to ' 'file %s\n' % (part.filename, mime, path)) # Pokud je tento soubor j*z v databazi, zaznam znovu nepridavame try: file_in_db = session.query(model.SubmittedFile).\ filter(model.SubmittedFile.evaluation == evaluation.id).\ filter(model.SubmittedFile.path == path).scalar() if file_in_db is None: submitted_file = model.SubmittedFile( evaluation=evaluation.id, mime=mime, path=path) session.add(submitted_file) except SQLAlchemyError: session.rollback() raise evaluation.full_report = report try: session.add(evaluation) session.commit() except SQLAlchemyError: session.rollback() raise finally: session.close() req.context['result'] = {'result': 'ok'}
def on_post(self, req, resp): try: userinfo = req.context['user'] if not userinfo.is_logged_in(): resp.status = falcon.HTTP_400 return user = session.query(model.User).\ filter(model.User.id == userinfo.get_id()).\ first() files = multipart.MultiDict() content_type, options = multipart.parse_options_header( req.content_type ) boundary = options.get('boundary', '') if not boundary: raise multipart.MultipartError("No boundary for " "multipart/form-data.") for part in multipart.MultipartParser(req.stream, boundary, req.content_length): files[part.name] = part file = files.get('file') user_id = req.context['user'].get_id() tmpfile = tempfile.NamedTemporaryFile(delete=False) file.save_as(tmpfile.name) mime = magic.Magic(mime=True).from_file(tmpfile.name) if mime not in ALLOWED_MIME_TYPES: resp.status = falcon.HTTP_400 return if not os.path.isdir(UPLOAD_DIR): try: os.makedirs(UPLOAD_DIR) except OSError: print('Unable to create directory for profile pictures') resp.status = falcon.HTTP_500 return new_picture = os.path.join(UPLOAD_DIR, 'user_%d.%s' % ( user_id, ALLOWED_MIME_TYPES[mime] )) self._crop(tmpfile.name, new_picture) try: os.remove(tmpfile.name) except OSError: print('Unable to remove temporary file %s' % tmpfile.name) user.profile_picture = new_picture session.commit() req.context['result'] = {} except SQLAlchemyError: session.rollback() raise finally: session.close()