def test_iterlines_maxbuf(self):
data, limit = 'abcdefgh\nijklmnop\r\nq', 9
result = [(to_bytes('abcdefgh'),to_bytes('\n')),(to_bytes('ijklmnop'),to_bytes('')),(to_bytes(''),to_bytes('\r\n')),(to_bytes('q'),to_bytes(''))]
i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo', buffer_size=limit)._lineiter()
self.assertEqual(list(i), result)
data, limit = ('X'*3*1024)+'x\n', 1024
result = [(to_bytes('X'*1024),to_bytes('')),(to_bytes('X'*1024),to_bytes('')),(to_bytes('X'*1024),to_bytes('')),(to_bytes('x'),to_bytes('\n'))]
i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo', buffer_size=limit)._lineiter()
self.assertEqual(list(i), result)
def test_iterlines_limit(self):
data, limit = 'abc\ndef\r\nghi', 10
result = [(to_bytes('abc'),to_bytes('\n')),(to_bytes('def'),to_bytes('\r\n')),(to_bytes('g'),to_bytes(''))]
i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo', limit)._lineiter()
self.assertEqual(list(i), result)
data, limit = 'abc\ndef\r\nghi', 8
result = [(to_bytes('abc'),to_bytes('\n')),(to_bytes('def'),to_bytes('\r'))]
i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo', limit)._lineiter()
self.assertEqual(list(i), result)
def decode_multiform_data(data):
s_pat = data.split(b"\r")[0][2:]
parts = multipart.MultipartParser(BytesIO(multipart.to_bytes(data)),
s_pat).parts()
d_d = {}
for part_t in parts:
if part_t.content_type != '':
d_d[part_t.name] = {
'isfile': True,
'filename': part_t.filename,
'name': part_t.name,
'file': part_t.file,
'size': part_t.size,
}
else:
try:
value = tornado.escape.json_decode(part_t.value)
except:
value = part_t.value
# endtry
d_d[part_t.name] = {
'isfile': False,
'name': part_t.name,
'value': value,
}
# endif
# endfor
return d_d
def test_big_file(self):
''' If the size of an uploaded part exceeds memfile_limit,
it is written to disk. '''
test_file = 'abc' * 1024
boundary = '---------------------------186454651713519341951581030105'
request = BytesIO(
to_bytes('\r\n').join(
map(to_bytes, [
'--' + boundary,
'Content-Disposition: form-data; name="file1"; filename="random.png"',
'Content-Type: image/png', '', test_file, '--' + boundary,
'Content-Disposition: form-data; name="file2"; filename="random.png"',
'Content-Type: image/png', '', test_file + 'a',
'--' + boundary,
'Content-Disposition: form-data; name="file3"; filename="random.png"',
'Content-Type: image/png', '', test_file * 2,
'--' + boundary + '--', ''
])))
p = mp.MultipartParser(request, boundary, memfile_limit=len(test_file))
self.assertEqual(p.get('file1').file.read(), to_bytes(test_file))
self.assertTrue(p.get('file1').is_buffered())
self.assertEqual(p.get('file2').file.read(), to_bytes(test_file + 'a'))
self.assertFalse(p.get('file2').is_buffered())
self.assertEqual(p.get('file3').file.read(), to_bytes(test_file * 2))
self.assertFalse(p.get('file3').is_buffered())
def test_save_as(self):
''' save_as stores data in a file keeping the file position. '''
def tmp_file_name():
# create a temporary file name (on Python 2.6+ NamedTemporaryFile
# with delete=False could be used)
fd, fname = tempfile.mkstemp()
f = os.fdopen(fd)
f.close()
return fname
test_file = 'abc' * 1024
boundary = '---------------------------186454651713519341951581030105'
request = BytesIO(
to_bytes('\r\n').join(
map(to_bytes, [
'--' + boundary,
'Content-Disposition: form-data; name="file1"; filename="random.png"',
'Content-Type: image/png', '', test_file,
'--' + boundary + '--', ''
])))
p = mp.MultipartParser(request, boundary)
self.assertEqual(
p.get('file1').file.read(1024),
to_bytes(test_file)[:1024])
tfn = tmp_file_name()
p.get('file1').save_as(tfn)
tf = open(tfn, 'rb')
self.assertEqual(tf.read(), to_bytes(test_file))
tf.close()
self.assertEqual(
p.get('file1').file.read(),
to_bytes(test_file)[1024:])
def post_file_callback(request):
s = request.body.split(b"\r")[0][2:]
p = list(
multipart.MultipartParser(BytesIO(multipart.tob(request.body)),
s))[0]
assert "filename" in p.options
return 200, JSON_HEADER, json.dumps({"test": "test"})
def test_iterlines(self):
data = 'abc\ndef\r\nghi'
result = [(to_bytes('abc'), to_bytes('\n')),
(to_bytes('def'), to_bytes('\r\n')),
(to_bytes('ghi'), to_bytes(''))]
i = mp.MultipartParser(BytesIO(to_bytes(data)), 'foo')._lineiter()
self.assertEqual(list(i), result)
def do_POST(self):
host = self.client_address[0]
try:
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
stream = BytesIO(body)
boundary = stream.readline()
boundary = boundary.strip(b"\r\n")[2:]
stream.seek(0)
parser = multipart.MultipartParser(stream, boundary)
for part in parser:
# Try not to allow uploads to things outside of upload_dir.
# WARNING: probably not secure
localpath = os.path.join(
args.upload_dir,
os.path.basename(os.path.realpath(part.name)))
print("Submission {} uploading to {}".format(
part.name, localpath))
with open(localpath, 'wb') as f:
shutil.copyfileobj(part.file, f)
except Exception as e:
print(e)
self._reply(str(e).encode())
else:
self._reply(b"Upload successful!\n")
def bytes_to_fragments(b):
ret = []
bio = io.BytesIO(b)
parser = multipart.MultipartParser(bio, b"MIME-BOUNDARY", charset=None)
for part in parser:
ret.append(Fragment(part.name, part.raw))
return ret
def read_form_data(self):
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length).decode("utf8")
parser = multipart.MultipartParser(
BytesIO(multipart.to_bytes(post_data)),
post_data.split("\r")[0][2:])
parts = parser.parts()
return {part.name: part.value for part in parts}
def test_file_seek(self):
''' The file object should be readable withoud a seek(0). '''
test_file = 'abc'*1024
boundary = '---------------------------186454651713519341951581030105'
request = BytesIO(to_bytes('\r\n').join(map(to_bytes,[
'--' + boundary,
'Content-Disposition: form-data; name="file1"; filename="random.png"',
'Content-Type: image/png', '', test_file, '--' + boundary + '--',''])))
p = mp.MultipartParser(request, boundary)
self.assertEqual(p.get('file1').file.read(), to_bytes(test_file))
self.assertEqual(p.get('file1').value, test_file)
def test_unicode_value(self):
''' The .value property always returns unicode '''
test_file = 'abc'*1024
boundary = '---------------------------186454651713519341951581030105'
request = BytesIO(to_bytes('\r\n').join(map(to_bytes,[
'--' + boundary,
'Content-Disposition: form-data; name="file1"; filename="random.png"',
'Content-Type: image/png', '', test_file, '--' + boundary + '--',''])))
p = mp.MultipartParser(request, boundary)
self.assertEqual(p.get('file1').file.read(), to_bytes(test_file))
self.assertEqual(p.get('file1').value, test_file)
self.assertTrue(hasattr(p.get('file1').value, 'encode'))
def do_upload(self):
# Do some browsers /really/ use multipart ? maybe Opera ?
try:
#self.log_message("Started file transfer")
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
print("Downloading {} bytes...".format(content_length))
#print("Content Type: {}".format(self.headers.get_content_type()))
#print("Body: {}".format(body))
stream = BytesIO(body)
boundary = stream.readline()
boundary = boundary.strip(b"\r\n")[2:]
#print("Boundary: {}".format(boundary))
stream.seek(0)
parser = multipart.MultipartParser(stream, boundary)
#print("{}".format(parser))
#print("Data:\n{}".format(parser.get('data').file.read()))
for part in parser:
host = self.get_host().replace(":", "_")
#print("Joining {}+{}+{}".format(SETTINGS['exfil_dir'], host , part.name))
localpath = os.path.join(SETTINGS['exfil_dir'], host,
part.name.lstrip("/"))
#localpath = os.path.join(localpath , part.name) #.encode('utf-8')
#print("Initial path: {}".format(localpath))
root, ext = os.path.splitext(localpath)
i = 1
try:
os.makedirs(os.path.dirname(localpath), 755)
except Exception as e:
pass
# race condition, but hey...
while (os.path.exists(localpath)):
localpath = "%s-%d%s" % (root, i, ext)
i = i + 1
#print("Writing to: {}".format(localpath))
fout = open(localpath, 'wb')
shutil.copyfileobj(part.file, fout)
fout.close()
os.chmod(localpath, 755)
#self.log_message("Received: %s", os.path.basename(localpath))
#self.send_html(self.html("success"))
print("Downloaded: {}".format(localpath))
except Exception as e:
#self.log_message(repr(e))
print("Error downloading: {}".format(e))
traceback.print_exc()
async def parse_formdata(body_type, request: aiohttp.web.BaseRequest) -> dict:
indata = {}
for key, val in urllib.parse.parse_qsl(request.query_string):
indata[key] = val
# PUT/POST form data?
if request.method in ["PUT", "POST"]:
if request.can_read_body:
try:
if (request.content_length
and request.content_length > PYPONY_MAX_PAYLOAD):
raise ValueError(
"Form data payload too large, max 256kb allowed")
body = await request.text()
if body_type == "json":
try:
js = json.loads(body)
assert isinstance(
js, dict
) # json data MUST be an dictionary object, {...}
indata.update(js)
except ValueError:
raise ValueError("Erroneous payload received")
elif body_type == "form":
if (request.headers.get("content-type", "").lower() ==
"application/x-www-form-urlencoded"):
try:
for key, val in urllib.parse.parse_qsl(body):
indata[key] = val
except ValueError:
raise ValueError("Erroneous payload received")
# If multipart, turn our body into a BytesIO object and use multipart on it
elif ("multipart/form-data"
in request.headers.get("content-type", "").lower()):
fh = request.headers.get("content-type", "")
fb = fh.find("boundary=")
if fb > 0:
boundary = fh[fb + 9:]
if boundary:
try:
for part in multipart.MultipartParser(
io.BytesIO(body.encode("utf-8")),
boundary,
len(body),
):
indata[part.name] = part.value
except ValueError:
raise ValueError(
"Erroneous payload received")
finally:
pass
return indata
def upload_file():
if request.method == 'POST':
data = request.data
#s = str(data).split("\r")[0][16:]
s = data[2:42]
print("s is ", s)
p = mp.MultipartParser(BytesIO(tob(data)),s)
newFile = open("/tmp/screenshot.png", "wb")
#newFile.write(data)
newFile.write((p.parts()[0].value.encode("latin-1")))
#print("Data size: {}".format(len(request.data)))
return "OK"
def resolve_multipart(data) -> dict:
boundary = data.decode().split("\r")[0][2:]
multipart_parser = mp.MultipartParser(BytesIO(data), boundary)
blob = multipart_parser.parts()[0].value
temp_file_name = tempfile.NamedTemporaryFile(delete=False).name
f = open(temp_file_name, "wb")
f.write(blob.encode("latin-1"))
f.close()
parameters = multipart_parser.parts()[1].value
parameters_json = yaml.load(parameters, Loader=yaml.SafeLoader)
parameters_json['workflow_yaml'] = temp_file_name
return parameters_json
def test_get_all(self):
''' Test the get() and get_all() methods. '''
boundary = '---------------------------186454651713519341951581030105'
request = BytesIO(to_bytes('\r\n').join(map(to_bytes,[
'--' + boundary,
'Content-Disposition: form-data; name="file1"; filename="random.png"',
'Content-Type: image/png', '', 'abc'*1024, '--' + boundary,
'Content-Disposition: form-data; name="file1"; filename="random.png"',
'Content-Type: image/png', '', 'def'*1024, '--' + boundary + '--',''])))
p = mp.MultipartParser(request, boundary)
self.assertEqual(p.get('file1').file.read(), to_bytes('abc'*1024))
self.assertEqual(p.get('file2'), None)
self.assertEqual(len(p.get_all('file1')), 2)
self.assertEqual(p.get_all('file1')[1].file.read(), to_bytes('def'*1024))
self.assertEqual(p.get_all('file1'), p.parts())
def parse_body(body, ctype, projname, dm_type):
body_n = None
if "multipart/form-data" in ctype:
# body = str(body) # COMMENT: old code line
body = body.encode("utf-8") # COMMENT: code line that does not run into encoding error
# COMMENT: parse multipart/form-data body
body_n = ParseTree(projname, MULTIPART, body)
# s_obj = StringIO(body.encode("utf-8")) # COMMENT: old code line
s_obj = StringIO(body) # new code line reflecting previous changes
boundary = ctype.split("; boundary=")[1]
mp = multipart.MultipartParser(s_obj, boundary)
pos = 0
for part in mp:
k = part.options.get("name")
v = part.value
kv_n = PTNonTerminalNode(projname, MULTIPART, "multipart-pair", pos)
k_n = PTTerminalNode(projname, dm_type, k, "multipart-name", 0)
v_n = PTTerminalNode(projname, dm_type, v, "multipart-value", 1)
kv_n.HasChild.add(k_n)
kv_n.HasChild.add(v_n)
body_n.HasChild.add(kv_n)
pos += 1
elif "application/x-www-form-urlencoded" in ctype:
body = str(body)
body_n = ParseTree(projname, FORMURLENC, body)
pos = 0
for k, vs in parse_qs(body).iteritems():
for v in vs:
pair_n = PTNonTerminalNode(projname, URL, "form-urlenc-pair", pos)
pair_n.HasChild.add(PTTerminalNode(projname, FORMURLENC, k, "form-urlenc-param-name", 0))
pair_n.HasChild.add(PTTerminalNode(projname, FORMURLENC, v, "form-urlenc-param-value", 1))
body_n.HasChild.add(pair_n)
pos += 1
elif "json" in ctype:
body = str(body)
body_n = ParseTree(projname, JSON, body)
cnt = visit_json(projname, json.loads(body))
body_n.HasChild.add(cnt)
else:
body_n = ParseTree(projname, ctype, "{} file, we ignore this content".format(ctype))
s_n = PTTerminalNode(projname, dm_type, "{} file, we ignore this content".format(ctype), "plaintext-body", 0)
body_n.HasChild.add(s_n)
return body_n
def do_POST(self):
try:
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
stream = BytesIO(body)
boundary = stream.readline()
boundary = boundary.strip(b"\r\n")[2:]
stream.seek(0)
parser = multipart.MultipartParser(stream, boundary)
for part in parser:
res = part.file.read().decode()
if res:
print(res)
except Exception as e:
print(e)
def nbformathandler(event, context):
headers = event['headers'];
content_type = headers['content-type']
boundary = content_type.split('=')[1]
httpbody = mp.MultipartParser(BytesIO(str.encode(event['body'])), boundary)
data = httpbody.get('data').file.read()
in_memory_source = BytesIO(data)
res = execute_notebook(in_memory_source)
response = {
"statusCode": 200,
"body": res
}
return response
def test_multiline_header(self):
''' HTTP allows headers to be multiline. '''
test_file = to_bytes('abc'*1024)
test_text = u'Test text\n with\r\n ümläuts!'
boundary = '---------------------------186454651713519341951581030105'
request = BytesIO(to_bytes('\r\n').join(map(to_bytes,[
'--' + boundary,
'Content-Disposition: form-data;',
'\tname="file1"; filename="random.png"',
'Content-Type: image/png', '', test_file, '--' + boundary,
'Content-Disposition: form-data;',
' name="text"', '', test_text,
'--' + boundary + '--',''])))
p = mp.MultipartParser(request, boundary, charset='utf8')
self.assertEqual(p.get('file1').file.read(), test_file)
self.assertEqual(p.get('file1').filename, 'random.png')
self.assertEqual(p.get('text').value, test_text)
def do_POST(self):
if self.path.lower().strip('/') != str(guid):
self.send_error(HTTPStatus.NOT_FOUND, 'File not found')
return None
# Receive and store file
content_length = int(self.headers['Content-Length'])
_, options = cgi.parse_header(self.headers['Content-Type'])
boundary = options['boundary']
data = io.BytesIO(self.rfile.read(content_length))
parser = multipart.MultipartParser(data, boundary)
selector_data = None
for part in parser.parts():
if part.name == 'fileselector':
selector_data = part.raw
filename = part.filename
break
else:
print('Error: Data not found')
self.send_error(HTTPStatus.BAD_REQUEST,
'Improperly formed request')
new_file_path = file_path / filename
if new_file_path.exists():
self.send_response(HTTPStatus.OK)
message = FAIL_FORM_HTML.format('File already exists').encode()
else:
with open(file_path / filename, 'wb') as f:
f.write(selector_data)
print(f'Received {filename}')
self.send_response(HTTPStatus.CREATED)
message = SUCCESS_FORM_HTML.encode()
self.send_header("Content-type", 'text/html')
self.send_header("Content-Length", str(len(message)))
self.end_headers()
self.wfile.write(message)
self.wfile.flush()
def _multipart_form_parser(request):
_, options = cgi.parse_header(request['content_type'])
boundary = options['boundary']
parser = multipart.MultipartParser(request.body, boundary)
multipart_params = m().evolver()
form_params = m().evolver()
for part in parser.parts():
if part.disposition == 'form-data' and not part.content_type:
form_params[part.name] = part.value
multipart_params[part.name] = m(
content_type=part.content_type or None,
content_length=part.size,
headers=pmap(part.headerlist),
name=part.name,
filename=part.filename,
character_encoding=part.charset,
body=part.file)
request = request.transform(
['form_params'],
lambda params: params.update(form_params.persistent()))
return request.set('multipart_params', multipart_params.persistent())
def decode_multipart(o, content_type, **kwargs):
har = []
try:
if isinstance(o, six.string_types):
import multipart
content_type, options = multipart.parse_options_header(
content_type)
assert content_type == 'multipart/form-data'
stream = six.BytesIO(o)
boundary = six.binary_type(options.get('boundary'))
assert boundary
for part in multipart.MultipartParser(stream, boundary, len(o),
**kwargs):
if part.filename or not part.is_buffered():
param = {'name': part.name, 'value': part.value,
'filename': part.filename}
else: # TODO: Big form-fields are in the files dict. really?
param = {'name': part.name, 'value': part.value}
har.append(param)
except Exception as err:
print(repr(err))
return har
def test_line_parser(self):
for line in ('foo', ''):
for ending in ('\n', '\r', '\r\n'):
i = mp.MultipartParser(BytesIO(to_bytes(line + ending)), 'foo')
i = next(i._lineiter())
self.assertEqual(i, (to_bytes(line), to_bytes(ending)))
async def parse(self) -> FormData:
# Parse the Content-Type header to get the multipart boundary.
content_type, params = parse_options_header(self.headers["Content-Type"])
boundary = params.get(b"boundary")
# Callbacks dictionary.
callbacks = {
"on_part_begin": self.on_part_begin,
"on_part_data": self.on_part_data,
"on_part_end": self.on_part_end,
"on_header_field": self.on_header_field,
"on_header_value": self.on_header_value,
"on_header_end": self.on_header_end,
"on_headers_finished": self.on_headers_finished,
"on_end": self.on_end,
}
# Create the parser.
parser = multipart.MultipartParser(boundary, callbacks)
header_field = b""
header_value = b""
raw_headers = [] # type: typing.List[typing.Tuple[bytes, bytes]]
field_name = ""
data = b""
file = None # type: typing.Optional[UploadFile]
items = (
[]
) # type: typing.List[typing.Tuple[str, typing.Union[str, UploadFile]]]
# Feed the parser with data from the request.
async for chunk in self.stream:
parser.write(chunk)
messages = list(self.messages)
self.messages.clear()
for message_type, message_bytes in messages:
if message_type == MultiPartMessage.PART_BEGIN:
raw_headers = []
data = b""
elif message_type == MultiPartMessage.HEADER_FIELD:
header_field += message_bytes
elif message_type == MultiPartMessage.HEADER_VALUE:
header_value += message_bytes
elif message_type == MultiPartMessage.HEADER_END:
raw_headers.append((header_field.lower(), header_value))
header_field = b""
header_value = b""
elif message_type == MultiPartMessage.HEADERS_FINISHED:
headers = Headers(raw=raw_headers)
content_disposition = headers.get("Content-Disposition")
content_type = headers.get("Content-Type", "")
disposition, options = parse_options_header(content_disposition)
field_name = options[b"name"].decode("latin-1")
if b"filename" in options:
filename = options[b"filename"].decode("latin-1")
file = UploadFile(filename=filename, content_type=content_type)
else:
file = None
elif message_type == MultiPartMessage.PART_DATA:
if file is None:
data += message_bytes
else:
await file.write(message_bytes)
elif message_type == MultiPartMessage.PART_END:
if file is None:
items.append((field_name, data.decode("latin-1")))
else:
await file.seek(0)
items.append((field_name, file))
elif message_type == MultiPartMessage.END:
pass
parser.finalize()
return FormData(items)
def opentc_REQMOD(self):
self.multipart_data = None
self.last_form_field = None
self.big_chunk = b''
self.content_analysis_results = dict()
try:
response = self.server.opentc["client"].command("PING\n")
response = json.loads(response.decode('utf-8'))
self.logger.debug("REQMOD Ping response: {}".format(response))
except Exception as err:
self.logger.error(traceback.format_exc())
def on_part_begin():
self.multipart_data = dict()
self.multipart_data[b'Content'] = b''
self.logger.debug("on_part_begin")
def on_part_data(data, start, end):
self.multipart_data[b'Content'] += data[start:end]
self.logger.debug("on_part_data")
def on_part_end():
self.logger.debug("on_part_end")
for key in self.multipart_data:
if key == b'Content':
mime_type = magic.from_buffer(
self.multipart_data[b'Content'], mime=True)
self.logger.debug(
"Content mime_type: {}".format(mime_type))
if b'Content-Type' in self.multipart_data:
# content_type = [ct.strip() for ct in self.multipart_data[b'Content-Type'].split(b';')]
content_type = [mime_type]
content_disposition = {'name': '', 'filename': ''}
for x in self.multipart_data[
b'Content-Disposition'].split(b';'):
if b'=' in x:
key, value = x.split(b'=')
key = key.decode("utf-8").strip(" \"")
value = value.decode("utf-8").strip(" \"")
content_disposition[key] = value
print(content_disposition)
result = self.content_analyse(
converter=self.server.opentc["config"]
["converter"],
content_disposition=content_disposition,
content_type=content_type,
content=self.multipart_data[b'Content'],
content_min_length=self.server.opentc["config"]
["content_min_length"],
client=self.server.opentc["client"])
name = self.multipart_data[
b'Content-Disposition'].split(b';')[1].split(
b'=')[1]
self.content_analysis_results[name.decode(
"utf-8").replace('"', '')] = result
else:
self.logger.debug("{}: {}".format(
key, self.multipart_data[key]))
return
def on_header_field(data, start, end):
self.last_form_field = data[start:end]
self.logger.debug("on_header_field")
def on_header_value(data, start, end):
self.multipart_data[self.last_form_field] = data[start:end]
self.logger.debug("on_header_value")
def on_end():
self.logger.debug("on_end")
self.set_icap_response(200)
# self.set_enc_request(b' '.join(self.enc_req))
for h in self.enc_req_headers:
for v in self.enc_req_headers[h]:
self.set_enc_header(h, v)
# Copy the request body (in case of a POST for example)
if not self.has_body:
self.set_enc_request(b' '.join(self.enc_req))
self.send_headers(False)
return
if self.preview:
prevbuf = b''
while True:
chunk = self.read_chunk()
if chunk == b'':
break
prevbuf += chunk
if self.ieof:
self.send_headers(True)
if len(prevbuf) > 0:
self.write_chunk(prevbuf)
self.write_chunk(b'')
return
self.cont()
self.set_enc_request(b' '.join(self.enc_req))
self.send_headers(True)
if len(prevbuf) > 0:
self.write_chunk(prevbuf)
while True:
chunk = self.read_chunk()
self.write_chunk(chunk)
if chunk == b'':
break
else:
# Parse the Content-Type header to get the multipart boundary.
content_type, params = parse_options_header(
self.enc_req_headers[b'content-type'][0])
boundary = params.get(b'boundary')
parser = None
if boundary is not None:
# Callbacks dictionary.
callbacks = {
'on_part_begin': on_part_begin,
'on_part_data': on_part_data,
'on_part_end': on_part_end,
'on_header_field': on_header_field,
'on_header_value': on_header_value,
'on_end': on_end
}
parser = multipart.MultipartParser(boundary, callbacks)
while True:
chunk = self.read_chunk()
if chunk == b'':
break
self.big_chunk += chunk
if boundary is not None:
size = len(self.big_chunk)
start = 0
while size > 0:
end = min(size, 1024 * 1024)
parser.write(self.big_chunk[start:end])
size -= end
start = end
else:
result = self.content_analyse(
converter=self.server.opentc["config"]["converter"],
content_type=content_type,
content=self.big_chunk,
content_min_length=self.server.opentc["config"]
["content_min_length"],
client=self.server.opentc["client"])
name = "text"
self.content_analysis_results[name] = result
is_allowed = True
for result in self.content_analysis_results:
if self.content_analysis_results[result] is None:
continue
for classifier in self.server.opentc["config"][
"classifier_status"]:
if self.server.opentc["config"]["classifier_status"][
classifier] is False:
continue
for restricted_class in self.server.opentc["config"][
"restricted_classes"]:
self.logger.debug(
"{}: result:{}, classifier:{}".format(
restricted_class, result, classifier))
if restricted_class in self.content_analysis_results[
result][classifier]:
is_allowed = False
break
else:
is_allowed = True
if is_allowed is True:
break
if is_allowed is False:
break
if is_allowed:
self.set_enc_request(b' '.join(self.enc_req))
self.send_headers(True)
self.write_chunk(self.big_chunk)
else:
content = json.dumps(self.content_analysis_results)
content = "result={}".format(content).encode("utf-8")
enc_req = self.enc_req[:]
enc_req[0] = self.server.opentc["config"][
"replacement_http_method"].encode("utf-8")
enc_req[1] = self.server.opentc["config"][
"replacement_url"].encode("utf-8")
self.set_enc_request(b' '.join(enc_req))
self.enc_headers[b"content-type"] = [
b"application/x-www-form-urlencoded"
]
self.enc_headers[b"content-length"] = [
str(len(content)).encode("utf-8")
]
self.send_headers(True)
self.write_chunk(content)
async def parse(self) -> FormData:
# Parse the Content-Type header to get the multipart boundary.
content_type, params = parse_options_header(
self.headers["Content-Type"])
charset = params.get(b"charset", "utf-8")
if type(charset) == bytes:
charset = charset.decode("latin-1")
boundary = params[b"boundary"]
# Callbacks dictionary.
callbacks = {
"on_part_begin": self.on_part_begin,
"on_part_data": self.on_part_data,
"on_part_end": self.on_part_end,
"on_header_field": self.on_header_field,
"on_header_value": self.on_header_value,
"on_header_end": self.on_header_end,
"on_headers_finished": self.on_headers_finished,
"on_end": self.on_end,
}
# Create the parser.
parser = multipart.MultipartParser(boundary, callbacks)
header_field = b""
header_value = b""
content_disposition = None
content_type = b""
field_name = ""
data = b""
file: typing.Optional[UploadFile] = None
items: typing.List[typing.Tuple[str, typing.Union[str,
UploadFile]]] = []
item_headers: typing.List[typing.Tuple[bytes, bytes]] = []
# Feed the parser with data from the request.
async for chunk in self.stream:
parser.write(chunk)
messages = list(self.messages)
self.messages.clear()
for message_type, message_bytes in messages:
if message_type == MultiPartMessage.PART_BEGIN:
content_disposition = None
content_type = b""
data = b""
item_headers = []
elif message_type == MultiPartMessage.HEADER_FIELD:
header_field += message_bytes
elif message_type == MultiPartMessage.HEADER_VALUE:
header_value += message_bytes
elif message_type == MultiPartMessage.HEADER_END:
field = header_field.lower()
if field == b"content-disposition":
content_disposition = header_value
elif field == b"content-type":
content_type = header_value
item_headers.append((field, header_value))
header_field = b""
header_value = b""
elif message_type == MultiPartMessage.HEADERS_FINISHED:
disposition, options = parse_options_header(
content_disposition)
field_name = _user_safe_decode(options[b"name"], charset)
if b"filename" in options:
filename = _user_safe_decode(options[b"filename"],
charset)
file = UploadFile(
filename=filename,
content_type=content_type.decode("latin-1"),
headers=Headers(raw=item_headers),
)
else:
file = None
elif message_type == MultiPartMessage.PART_DATA:
if file is None:
data += message_bytes
else:
await file.write(message_bytes)
elif message_type == MultiPartMessage.PART_END:
if file is None:
items.append(
(field_name, _user_safe_decode(data, charset)))
else:
await file.seek(0)
items.append((field_name, file))
parser.finalize()
return FormData(items)
def _upload_files(self, req, module, user_id, resp):
# Soubory bez specifikace delky neberem.
if not req.content_length:
resp.status = falcon.HTTP_411
req.context['result'] = {
'result': 'error',
'error': 'Nelze nahrát neukončený stream.'
}
return
# Prilis velke soubory neberem.
if req.content_length > util.config.MAX_UPLOAD_FILE_SIZE:
resp.status = falcon.HTTP_413
req.context['result'] = {
'result': 'error',
'error': 'Maximální velikost dávky je 20 MB.'
}
return
# Pokud uz existuji odevzdane soubory, nevytvarime nove
# evaluation, pouze pripojujeme k j*z existujicimu
try:
existing = util.module.existing_evaluation(module.id, user_id)
if len(existing) > 0:
evaluation = session.query(model.Evaluation).get(existing[0])
evaluation.time = datetime.datetime.utcnow()
report = evaluation.full_report
else:
report = (str(datetime.datetime.now()) +
' : === Uploading files for module id \'%s\' for '
'task id \'%s\' ===\n' % (module.id, module.task))
evaluation = model.Evaluation(user=user_id, module=module.id,
ok=True)
session.add(evaluation)
session.commit()
# Lze uploadovat jen omezeny pocet souboru.
file_cnt = session.query(model.SubmittedFile).\
filter(model.SubmittedFile.evaluation ==
evaluation.id).count()
if file_cnt > util.config.MAX_UPLOAD_FILE_COUNT:
resp.status = falcon.HTTP_400
req.context['result'] = {
'result': 'error',
'error': 'K řešení lze nahrát nejvýše 20 souborů.'
}
return
except SQLAlchemyError:
session.rollback()
raise
dir = util.module.submission_dir(module.id, user_id)
try:
os.makedirs(dir)
except OSError:
pass
if not os.path.isdir(dir):
resp.status = falcon.HTTP_400
req.context['result'] = {
'result': 'error',
'error': 'Chyba 42, kontaktuj orga.'
}
return
files = multipart.MultiDict()
content_type, options = multipart.parse_options_header(
req.content_type)
boundary = options.get('boundary', '')
if not boundary:
raise multipart.MultipartError(
"No boundary for multipart/form-data.")
for part in multipart.MultipartParser(req.stream, boundary,
req.content_length, 2**30, 2**20,
2**18, 2**16, 'utf-8'):
path = '%s/%s' % (dir, part.filename)
part.save_as(path)
mime = magic.Magic(mime=True).from_file(path)
report += (str(datetime.datetime.now()) +
' : [y] uploaded file: \'%s\' (mime: %s) to '
'file %s\n' % (part.filename, mime, path))
# Pokud je tento soubor j*z v databazi, zaznam znovu nepridavame
try:
file_in_db = session.query(model.SubmittedFile).\
filter(model.SubmittedFile.evaluation == evaluation.id).\
filter(model.SubmittedFile.path == path).scalar()
if file_in_db is None:
submitted_file = model.SubmittedFile(
evaluation=evaluation.id,
mime=mime,
path=path)
session.add(submitted_file)
except SQLAlchemyError:
session.rollback()
raise
evaluation.full_report = report
try:
session.add(evaluation)
session.commit()
except SQLAlchemyError:
session.rollback()
raise
finally:
session.close()
req.context['result'] = {'result': 'ok'}
def on_post(self, req, resp):
try:
userinfo = req.context['user']
if not userinfo.is_logged_in():
resp.status = falcon.HTTP_400
return
user = session.query(model.User).\
filter(model.User.id == userinfo.get_id()).\
first()
files = multipart.MultiDict()
content_type, options = multipart.parse_options_header(
req.content_type
)
boundary = options.get('boundary', '')
if not boundary:
raise multipart.MultipartError("No boundary for "
"multipart/form-data.")
for part in multipart.MultipartParser(req.stream, boundary,
req.content_length):
files[part.name] = part
file = files.get('file')
user_id = req.context['user'].get_id()
tmpfile = tempfile.NamedTemporaryFile(delete=False)
file.save_as(tmpfile.name)
mime = magic.Magic(mime=True).from_file(tmpfile.name)
if mime not in ALLOWED_MIME_TYPES:
resp.status = falcon.HTTP_400
return
if not os.path.isdir(UPLOAD_DIR):
try:
os.makedirs(UPLOAD_DIR)
except OSError:
print('Unable to create directory for profile pictures')
resp.status = falcon.HTTP_500
return
new_picture = os.path.join(UPLOAD_DIR, 'user_%d.%s' % (
user_id, ALLOWED_MIME_TYPES[mime]
))
self._crop(tmpfile.name, new_picture)
try:
os.remove(tmpfile.name)
except OSError:
print('Unable to remove temporary file %s' % tmpfile.name)
user.profile_picture = new_picture
session.commit()
req.context['result'] = {}
except SQLAlchemyError:
session.rollback()
raise
finally:
session.close()