async def _parse_request(self, resp):
try:
reader = MultipartReader.from_response(resp)
except KeyError:
self._log('Request is not of type multipart.', 'warning')
return None
else:
self._log('Multipart request received.')
# Loop through multipart data
metadata = None
while True:
part = await reader.next()
if part is None:
break
# Verify if part contains header
try:
content_type = part.headers[hdrs.CONTENT_TYPE]
except KeyError:
self._log('Parsed part did not contain content_type header.',
'warning')
continue
else:
# Verify if part is of type JSON
if content_type == 'application/json':
metadata = await part.json()
self._log('Parsed part of type JSON, stop parsing.')
break
else:
self._log('Parsed part not of type JSON.', 'warning')
return metadata
async def uploadUnknownPage(self, request):
reader = MultipartReader.from_response(request)
part0 = await reader.next() # should be parameters
if part0 is None: # weird error
return web.Response(status=406) # should have sent 3 parts
param = await part0.json()
if not validate_required_fields(
param,
[
"user", "token", "fileName", "order", "md5sum", "bundle",
"bundle_order"
],
):
return web.Response(status=400)
if not self.server.validate(param["user"], param["token"]):
return web.Response(status=401)
if not param["user"] in ("manager", "scanner"):
return web.Response(status=401)
part1 = await reader.next() # should be the image file
if part1 is None: # weird error
return web.Response(status=406) # should have sent 3 parts
image = await part1.read()
# file it away.
rmsg = self.server.addUnknownPage(
param["fileName"],
image,
param["order"],
param["md5sum"],
param["bundle"],
param["bundle_order"],
)
return web.json_response(rmsg, status=200) # all good
async def foo(request):
reader = MultipartReader.from_response(request)
while True:
part = await reader.next()
if part is None:
break
if part.name == 'file':
contents = await part.read() #Тута вся твоя инфа
resp = asyncio.run_in_executor(None, bar, contents)
return web.Response(text=resp)
async def uploadCollidingPage(self, request):
reader = MultipartReader.from_response(request)
part0 = await reader.next() # should be parameters
if part0 is None: # weird error
return web.Response(status=406) # should have sent 2 parts
param = await part0.json()
if not validate_required_fields(
param,
[
"user",
"token",
"fileName",
"md5sum",
"test",
"page",
"version",
"bundle",
"bundle_order",
],
):
return web.Response(status=400)
if not self.server.validate(param["user"], param["token"]):
return web.Response(status=401)
# TODO - restrict to manager only.
if not param["user"] in ("manager", "scanner"):
return web.Response(status=401)
# TODO: unused, we should ensure this matches the data
code = request.match_info["tpv"]
part1 = await reader.next() # should be the image file
if part1 is None: # weird error
return web.Response(status=406) # should have sent 2 parts
image = await part1.read()
# file it away.
rmsg = self.server.addCollidingPage(
param["test"],
param["page"],
param["version"],
param["fileName"],
image,
param["md5sum"],
param["bundle"],
param["bundle_order"],
)
# note 200 used here for errors too
return web.json_response(rmsg, status=200)
async def image_handle(self, request):
if request.method == 'POST':
ret = {}
try:
reader = MultipartReader.from_response(request)
while True:
part = await reader.next()
if part is None:
break
data = await part.read()
data = np.asarray(data, dtype=np.uint8)
img = cv2.imdecode(data, cv2.IMREAD_COLOR)
ret = ofd.predict(img)
except web.HTTPException as e:
return self.json_error(e.reason)
return web.json_response(ret)
async def post(self):
''' Upload file from frontend to xls_file_path, parse it using Pandas and add to DB '''
xls_file_path='modules/file.xls'
reader = MultipartReader.from_response(self.request)
client_file=list()
while True:
part = await reader.next()
if part is None:
break
client_file.append(await part.read(decode=False))
with open(xls_file_path,'wb') as xls_file:
for part in client_file:
xls_file.write(part)
if os.path.exists(xls_file_path):
device_list = xls_to_base(xls_file_path)
os.remove(xls_file_path)
return web.json_response(device_list, content_type='application/json')
else:
return web.json_response('Error: File NOT created' , content_type='application/json')
async def multipart_parser(request, file_handler=default_file_handler):
"""
:param file_handler: callable to save file, this should always return the file path
:return: dictionary containing files and data
"""
multipart_data = {'files': {}, 'data': {}}
if request.content_type == 'multipart/form-data':
reader = MultipartReader.from_response(request)
while True:
part = await reader.next()
if part is None:
break
if isinstance(part, BodyPartReader):
if part.filename:
# if body is binary file
if file_handler:
# in case we just want to parse data and not save file actually e.g. in validator
file_data = await part.read(decode=True)
file_data = part.decode(file_data)
file_path = await file_handler(
part.filename, file_data,
part.headers[CONTENT_TYPE])
else:
file_path = part.filename
multipart_data['files'][part.name] = file_path
elif part.text():
# if body is text
text = await part.text()
multipart_data['data'][part.name] = text
else:
# if body is json or form (not text), not handling this
continue
else:
# if part is recursive multipart , not handling this right now
# TODO: do recursive call to handle this
continue
else:
try:
multipart_data['data'] = await request.json()
except json.JSONDecodeError:
pass
return multipart_data
async def add(request: web.Request) -> web.Response:
async with request.app['db'].acquire() as conn:
reader = MultipartReader.from_response(request)
request_data = {}
while True:
part = await reader.next()
if part is None:
break
logger.debug(f"part_name: {part.name}")
if hdrs.CONTENT_TYPE in part.headers and part.headers[
hdrs.CONTENT_TYPE].startswith('image'):
metadata = await part.read()
filename = get_random_filename()
filename += get_file_format(part.headers[hdrs.CONTENT_TYPE])
db_path = '/static/' + filename
path = ROOT_DIR + db_path
with open(path, "wb") as f:
f.write(metadata)
request_data[part.name] = db_path
else:
metadata = await part.text()
request_data[part.name] = metadata
current_time = datetime.now().isoformat()
request_data['created_at'] = current_time
request_data['last_updated'] = current_time
schema = PostRequestSchema(strict=True)
request = schema.load(request_data).data
category_id = await get_category_by_title(conn, request['category'])
request['category_id'] = category_id
await add_post(conn, request)
return web.json_response("Ok")
async def post_request(request):
if request.method == 'POST':
# initialize the data dictionary that will be returned from the
# view
data = {"success": False}
reader = MultipartReader.from_response(request)
while True:
part = await reader.next()
if part is None:
break
if part.headers[hdrs.CONTENT_TYPE] == 'application/json':
metadata = await part.json()
continue
filedata = await part.read(decode=False)
# ensure an image was properly uploaded to our endpoint
# read the image in PIL format
image = Image.open(io.BytesIO(filedata))
# preprocess the image and prepare it for classification
image = prepare_image(image, target=(224, 224))
# model = load_model()
# classify the input image and then initialize the list
# of predictions to return to the client
preds = model.predict(image)
results = imagenet_utils.decode_predictions(preds)
data["predictions"] = []
# loop over the results and add them to the list of
# returned predictions
for (imagenetID, label, prob) in results[0]:
r = {"label": label, "probability": float(prob)}
data["predictions"].append(r)
# indicate that the request was a success
data["success"] = True
return web.json_response(data)
async def execute(self, request: Request) -> StreamResponse:
temp_dir: Optional[str] = None
try:
ctx = self.cli_context_from_request(request)
if request.content_type.startswith("text"):
command = (await request.text()).strip()
elif request.content_type.startswith("multipart"):
command = request.headers["Resoto-Shell-Command"].strip()
temp = tempfile.mkdtemp()
temp_dir = temp
files = {}
# for now we assume that all multi-parts are file uploads
async for part in MultipartReader(request.headers,
request.content):
name = part.name
if not name:
raise AttributeError(
"Multipart request: content disposition name is required!"
)
path = os.path.join(
temp,
rnd_str()) # use random local path to avoid clashes
files[name] = path
with open(path, "wb") as writer:
while not part.at_eof():
writer.write(await part.read_chunk())
ctx = replace(ctx, uploaded_files=files)
else:
raise AttributeError(
f"Not able to handle: {request.content_type}")
# we want to eagerly evaluate the command, so that parse exceptions will throw directly here
parsed = await self.cli.evaluate_cli_command(command, ctx)
return await self.execute_parsed(request, command, parsed)
finally:
if temp_dir:
shutil.rmtree(temp_dir)
async def MreturnMarkedTask(self, request):
"""Save the graded/processes task, extract data and save to database.
This function also responds with the number of done tasks and the total number of tasks.
The returned statement is similar to MprogressCount.
Respond with status 200/400/401/406/409/410.
Log activity.
Args:
request (aiohttp.web_request.Request): Request of type
PUT /MK/tasks/`question code` which includes a multipart
object indication the marked test data. This request will
include 3 parts including [metadata, image, plom-file].
Returns:
aiohttp.web_response.Response: Responses with a list including the number of
graded tasks and the overall number of tasks.
"""
log_request("MreturnMarkedTask", request)
reader = MultipartReader.from_response(request)
# Dealing with the metadata.
task_metadata_object = await reader.next()
if task_metadata_object is None: # weird error
return web.Response(status=406) # should have sent 3 parts
task_metadata = await task_metadata_object.json()
# Validate that the dictionary has these fields.
if not validate_required_fields(
task_metadata,
[
"user",
"token",
"comments",
"pg",
"ver",
"score",
"mtime",
"tags",
"md5sum",
"integrity_check",
"image_md5s",
],
):
return web.Response(status=400)
# Validate username and token.
if not self.server.validate(task_metadata["user"],
task_metadata["token"]):
return web.Response(status=401)
comments = task_metadata["comments"] # List of comments.
task_code = request.match_info["task"] # Task code.
# Note: if user isn't validated, we don't parse their binary junk
# TODO: is it safe to abort during a multi-part thing?
# Dealing with the image.
task_image_object = await reader.next()
if task_image_object is None: # weird error
return web.Response(status=406) # should have sent 3 parts
task_image = await task_image_object.read()
# Dealing with the plom_file.
plom_file_object = await reader.next()
if plom_file_object is None: # weird error
return web.Response(status=406) # should have sent 3 parts
plomdat = await plom_file_object.read()
marked_task_status = self.server.MreturnMarkedTask(
task_metadata["user"],
task_code,
int(task_metadata["pg"]),
int(task_metadata["ver"]),
int(task_metadata["score"]),
task_image,
plomdat,
comments,
int(task_metadata["mtime"]),
task_metadata["tags"],
task_metadata["md5sum"],
task_metadata["integrity_check"],
task_metadata["image_md5s"],
)
# marked_task_status = either [True, Num Done tasks, Num Totalled tasks] or [False] if error.
if marked_task_status[0]:
num_done_tasks = marked_task_status[1]
total_num_tasks = marked_task_status[2]
return web.json_response([num_done_tasks, total_num_tasks],
status=200)
else:
if marked_task_status[1] == "no_such_task":
log.warning(
"Returning with error 410 = {}".format(marked_task_status))
return web.Response(status=410)
elif marked_task_status[1] == "not_owner":
log.warning(
"Returning with error 409 = {}".format(marked_task_status))
return web.Response(status=409)
elif marked_task_status[1] == "integrity_fail":
log.warning(
"Returning with error 406 = {}".format(marked_task_status))
return web.Response(status=406)
else:
log.warning(
"Returning with error 400 = {}".format(marked_task_status))
return web.Response(status=400)
async def uploadTestPage(self, request):
"""A test page has known page, known paper number, usually QR-coded.
Typically the page is QR coded, and thus we know precisely what
paper number, what question and what page. We may not know the
student depending on whether it was prenamed or not.
Args:
request (aiohttp.web_request.Request)
Returns:
aiohttp.web_response.Response: JSON data directly from the
database call.
Note: this uses the `status=200` success return code for some
kinds of failures: it simply returns whatever data the DB gave
back as blob of json for the client to deal with. Thus, this
API call is not recommended outside of Plom.
"""
reader = MultipartReader.from_response(request)
part0 = await reader.next() # should be parameters
if part0 is None: # weird error
return web.Response(status=406) # should have sent 3 parts
param = await part0.json()
if not validate_required_fields(
param,
[
"user",
"token",
"test",
"page",
"version",
"fileName",
"md5sum",
"bundle",
"bundle_order",
],
):
return web.Response(status=400)
if not self.server.validate(param["user"], param["token"]):
return web.Response(status=401)
if not param["user"] in ("manager", "scanner"):
return web.Response(status=401)
# TODO: unused, we should ensure this matches the data
# TODO: or why bother passing those in to param?
code = request.match_info["tpv"]
part1 = await reader.next() # should be the image file
if part1 is None: # weird error
return web.Response(status=406) # should have sent 3 parts
image = await part1.read()
# file it away.
rmsg = self.server.addTestPage(
param["test"],
param["page"],
param["version"],
param["fileName"],
image,
param["md5sum"],
param["bundle"],
param["bundle_order"],
)
# note 200 used here for errors too
return web.json_response(rmsg, status=200)
async def uploadLPage(self, request):
"""A loose page is self-scanned, known student, but unknown question.
Typically the page is without QR codes. The uploader knows what
student it belongs to but not what question.
DEPRECATED? Perhaps on its way to deprecation if HW Pages become
more general in the future.
Args:
request (aiohttp.web_request.Request)
Returns:
aiohttp.web_response.Response: JSON data directly from the
database call.
Note: this uses the `status=200` success return code for some
kinds of failures: it simply returns whatever data the DB gave
back as blob of json for the client to deal with. Thus, this
API call is not recommended outside of Plom.
"""
reader = MultipartReader.from_response(request)
part0 = await reader.next() # should be parameters
if part0 is None: # weird error
return web.Response(status=406) # should have sent 3 parts
param = await part0.json()
if not validate_required_fields(
param,
[
"user",
"token",
"sid",
"order",
"fileName",
"md5sum",
"bundle",
"bundle_order",
],
):
return web.Response(status=400)
if not self.server.validate(param["user"], param["token"]):
return web.Response(status=401)
if not param["user"] in ("manager", "scanner"):
return web.Response(status=401)
part1 = await reader.next() # should be the image file
if part1 is None: # weird error
return web.Response(status=406) # should have sent 3 parts
image = await part1.read()
# file it away.
rmsg = self.server.addLPage(
param["sid"],
param["order"],
param["fileName"],
image,
param["md5sum"],
param["bundle"],
param["bundle_order"],
)
# note 200 used here for errors too
return web.json_response(rmsg, status=200)
async def uploadHWPage(self, request):
"""A homework page is self-scanned, known student, and known questions.
Typically the page is without QR codes. The uploader knows what
student it belongs to and what question(s). The order within the
question is somewhat known too, at least within its upload bundle.
Args:
request (aiohttp.web_request.Request)
Returns:
aiohttp.web_response.Response: JSON data directly from the
database call.
The requests data has a `question` field, which can be a scalar
or a list of questions we wish to upload too. Maybe the scalar
is deprecated?
TODO: force it to always be a list?
Note: this uses the `status=200` success return code for some
kinds of failures: it simply returns whatever data the DB gave
back as blob of json for the client to deal with. Thus, this
API call is not recommended outside of Plom.
"""
reader = MultipartReader.from_response(request)
part0 = await reader.next() # should be parameters
if part0 is None: # weird error
return web.Response(status=406) # should have sent 3 parts
param = await part0.json()
if not validate_required_fields(
param,
[
"user",
"token",
"sid",
"question",
"order",
"fileName",
"md5sum",
"bundle",
"bundle_order",
],
):
return web.Response(status=400)
if not self.server.validate(param["user"], param["token"]):
return web.Response(status=401)
if not param["user"] in ("manager", "scanner"):
return web.Response(status=401)
part1 = await reader.next() # should be the image file
if part1 is None: # weird error
return web.Response(status=406) # should have sent 3 parts
image = await part1.read()
# file it away.
rmsg = self.server.addHWPage(
param["sid"],
param["question"],
param["order"],
param["fileName"],
image,
param["md5sum"],
param["bundle"],
param["bundle_order"],
)
# note 200 used here for errors too
return web.json_response(rmsg, status=200)
async def service_submission(request: web.Request):
reader = MultipartReader.from_response(request)
data = None
filedata = None
# Read multipart email
while True:
part = await reader.next() # pylint: disable=not-callable
if part is None:
break
if part.headers[hdrs.CONTENT_TYPE] == "application/json":
data = await part.json()
continue
if part.headers[hdrs.CONTENT_TYPE] == "application/zip":
filedata = await part.read(decode=True)
# Validate max file size
maxsize = 10 * 1024 * 1024 # 10MB
actualsize = len(filedata)
if actualsize > maxsize:
raise web.HTTPRequestEntityTooLarge(maxsize, actualsize)
filename = part.filename
continue
raise web.HTTPUnsupportedMediaType(
reason=f"One part had an unexpected type: {part.headers[hdrs.CONTENT_TYPE]}"
)
# data (dict) and file (bytearray) have the necessary information to compose the email
support_email_address = request.app[APP_CONFIG_KEY]["smtp"]["sender"]
is_real_usage = any(
env in os.environ.get("SWARM_STACK_NAME", "")
for env in ("production", "staging")
)
db = get_storage(request.app)
user = await db.get_user({"id": request[RQT_USERID_KEY]})
user_email = user.get("email")
if not is_real_usage:
support_email_address = user_email
try:
# NOTE: temporarily internal import to avoid render_and_send_mail to be interpreted as handler
# TODO: Move outside when get_handlers_from_namespace is fixed
from .login.utils import render_and_send_mail
attachments = [("metadata.json", json.dumps(data, indent=4))]
if filedata:
attachments.append((filename, filedata))
# send email
await render_and_send_mail(
request,
to=support_email_address,
template=common_themed(EMAIL_TEMPLATE_NAME),
context={
"user": user_email,
"data": json2html.convert(
json=json.dumps(data), table_attributes='class="pure-table"'
),
"subject": "TEST: " * (not is_real_usage) + "New service submission",
},
attachments=attachments,
)
except Exception as exc:
log.exception("Error while sending the 'new service submission' mail.")
raise web.HTTPServiceUnavailable() from exc
raise web.HTTPNoContent(content_type="application/json")
