async def _upload_data(self, media_id: str,
data: bytes) -> MediaUploadResponse:
max_size = 2**17
base_upload_req = self.upload_url.with_query({
"command": "APPEND",
"media_id": media_id,
})
for i in range(math.ceil(len(data) / max_size)):
multipart_data = MultipartWriter("form-data")
part = multipart_data.append(data[i * max_size:(i + 1) * max_size])
part.set_content_disposition("form-data",
name="media",
filename="blob")
req = base_upload_req.update_query({"segment_index": i})
async with self.http.post(req,
data=multipart_data,
headers=self.headers) as resp:
await check_error(resp)
self.log.debug(f"Uploaded segment {i} of {media_id}")
finalize_req = self.upload_url.with_query({
"command": "FINALIZE",
"media_id": media_id,
})
async with self.http.post(finalize_req, headers=self.headers) as resp:
resp_data = await check_error(resp)
processing_info = resp_data.get("processing_info", {})
if processing_info.get("state", None) == "pending":
self.log.debug(
f"Finished uploading {media_id}, but server is still processing it"
)
check_after = processing_info.get("check_after_secs", 1)
return await self._wait_processing(media_id, check_after)
self.log.debug(f"Finished uploading {media_id}")
return MediaUploadResponse.deserialize(resp_data)
async def upload_file(url, username, password, file_path, runtime):
async with aiofiles.open(file_path, mode='rb') as f:
file_contents = await f.read()
responses = runtime['responses']
mpw = MultipartWriter()
part = mpw.append(file_contents)
part.set_content_disposition('attachment',
filename=path.basename(file_path))
fd = FormData()
fd.add_field('file', file_contents, filename=path.basename(file_path))
fd.add_field('username', username)
fd.add_field('password', password)
async with ClientSession() as session:
try:
async with session.post(url, data=fd) as response:
response = await response.read()
response_str = response.decode('utf-8')
if len(response_str) > 15:
response_str = response_str[:12] + "..."
responses[response_str] = responses.get(response_str, 0) + 1
except ClientConnectionError:
responses['CONNECTION_ERR'] = responses.get('CONNECTION_ERR',
0) + 1
except ClientPayloadError:
responses['PAYLOAD_ERR'] = responses.get('PAYLOAD_ERR', 0) + 1
except ClientResponseError:
responses['RESPONSE_ERR'] = responses.get('RESPONSE_ERR', 0) + 1
async def upload_file(url, username, password, file_path, responses):
async with aiofiles.open(file_path, mode='rb') as f:
file_contents = await f.read()
mpw = MultipartWriter()
part = mpw.append(file_contents)
part.set_content_disposition('attachment',
filename=path.basename(file_path))
form_data = {
'username': username,
'password': password,
'file': file_contents
}
async with ClientSession() as session:
try:
async with session.post(url, data=form_data) as response:
response = await response.read()
response_str = response.decode('utf-8')
responses[response_str] = responses.get(response_str, 0) + 1
except ClientConnectionError:
responses['CONNECTION_ERR'] = responses.get('CONNECTION_ERR',
0) + 1
except ClientPayloadError:
responses['PAYLOAD_ERR'] = responses.get('PAYLOAD_ERR', 0) + 1
except ClientResponseError:
responses['RESPONSE_ERR'] = responses.get('RESPONSE_ERR', 0) + 1
async def coroutine():
def extract_files(obj):
fs, idx = {}, 0
def rec(v):
if isinstance(v, list):
return [rec(x) for x in v]
if isinstance(v, dict):
return {k: rec(v) for k, v in v.items()}
nonlocal fs, idx
if isinstance(v, tuple) and len(v) == 2 and isinstance(
v[1], IOBase):
name = str(v[0])
fs[name] = v[1]
return 'attach://' + name
if isinstance(v, IOBase):
name = 'file_' + str(idx)
idx += 1
fs[name] = v
return 'attach://' + name
return v
return rec(obj), fs
args, files = extract_files(kwargs)
log = method not in self.trace_methods
if log:
logger.debug('calling method %s %s', method, args)
if files:
query = urlencode({
k: json.dumps(v) if isinstance(v, (list, dict)) else str(v)
for k, v in args.items()
})
writer = MultipartWriter('form-data')
for n, file in files.items():
writer.append(file).set_content_disposition('attachment',
filename=n)
response = await self.__session.post(self.__url + method +
'?' + query,
data=writer)
else:
response = await self.__session.post(self.__url + method,
json=args)
data = DynamicDictObject(await response.json())
if log:
logger.debug('received answer %s', data)
if 'ok' in data:
if data.ok and 'result' in data:
return data.result
if 'description' in data:
raise RequestError('server error calling \'%s\': %s' %
(method, data.description))
raise RequestError('bad response: %s' % data)
async def create_thread_in_forum(
self,
channel_id: int,
name: str,
auto_archive_duration: int,
message_payload: dict,
applied_tags: List[str] = None,
files: Optional[List[File]] = MISSING,
rate_limit_per_user: Optional[int] = None,
reason: Optional[str] = None,
) -> dict:
"""
From a given Forum channel, create a Thread with a message to start with.
:param channel_id: The ID of the channel to create this thread in
:param name: The name of the thread
:param auto_archive_duration: duration in minutes to automatically archive the thread after recent activity,
can be set to: 60, 1440, 4320, 10080
:param message_payload: The payload/dictionary contents of the first message in the forum thread.
:param files: An optional list of files to send attached to the message.
:param rate_limit_per_user: Seconds a user has to wait before sending another message (0 to 21600), if given.
:param reason: An optional reason for the audit log
:return: Returns a Thread in a Forum object with a starting Message.
"""
query = {
"has_message": "True"
} # TODO: Switch query after new feature breaking release.
payload = {"name": name, "auto_archive_duration": auto_archive_duration}
if rate_limit_per_user:
payload["rate_limit_per_user"] = rate_limit_per_user
if applied_tags:
payload["applied_tags"] = applied_tags
# payload.update(**{'use_nested_fields': 1})
data = None
if files is not MISSING and len(files) > 0:
data = MultipartWriter("form-data")
part = data.append_json(payload)
part.set_content_disposition("form-data", name="payload_json")
payload = None
for id, file in enumerate(files):
part = data.append(file._fp, )
part.set_content_disposition("form-data",
name=f"files[{str(id)}]",
filename=file._filename)
else:
payload.update(message_payload)
return await self._req.request(
Route("POST", f"/channels/{channel_id}/threads?has_message=True"),
json=payload,
data=data,
params=query,
reason=reason,
)
def _create_task(path, loop, semaphore, session):
default = "application/octet-stream"
mime_type = (content_type or mimetypes.guess_type(str(path))[0] or default)
# FIXME Nexus seems to not parse the Content-Disposition 'filename*' field properly.
# data = FormData()
# data.add_field("file", path.open("rb"), content_type=mime_type, filename=path.name)
# FIXME This hack is to prevent sending Content-Disposition with the 'filename*' field.
data = MultipartWriter("form-data")
part = data.append(path.open("rb"))
part.headers[CONTENT_TYPE] = mime_type
part.headers[CONTENT_DISPOSITION] = f'form-data; name="file"; filename="{path.name}"'
return loop.create_task(_upload(data, semaphore, session))
async def test_multiple_posts(cli_with_db, caplog):
caplog.set_level(logging.INFO)
manifest: OCIManifest = {
"schemaVersion":
2,
"config":
OCIContentDescriptor(
mediaType="application/vnd.oci.image.config.v1+json",
size=5555,
digest=
"sha256:0002b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537000",
annotations={},
urls=[],
),
"layers": [
OCIContentDescriptor(
mediaType="application/vnd.oci.image.layer.v1.tar+gzip",
size=77777,
digest=
"sha256:1234876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee123",
annotations={},
urls=["https://bitbucket/file1.tar.gz"],
),
],
"annotations": {},
"mediaType":
"",
}
with MultipartWriter("mixed") as mpwriter:
mpwriter.append_json(manifest)
resp = await cli_with_db.post("/manifest", data=mpwriter)
assert resp.status == 200, f"Error message {await resp.text()}"
message = await resp.json()
assert message["manifest_digest"] == manifest["config"]["digest"]
assert message["message"]
assert message["timestamp"]
with MultipartWriter("mixed") as mpwriter:
mpwriter.append_json(manifest)
resp = await cli_with_db.post("/manifest", data=mpwriter)
assert resp.status == 400
message = await resp.json()
assert message["manifest"] == manifest
assert message["message"]
assert message["error"]
def IDclaimThisTask(self, data, request):
"""Claims this identifying task and returns images of the ID pages.
Responds with status 200/204/404.
Args:
data (dict): A (str:str) dictionary having keys `user` and `token`.
request (aiohttp.web_request.Request): PATCH /ID/tasks request object.
Returns:
aiohttp.web_response.Response: A response including a aiohttp object which
includes a multipart object with the images.
"""
testNumber = request.match_info["task"]
image_path = self.server.IDclaimThisTask(
data["user"], testNumber) # returns [True, IMG_path] or [False]
allow_access = image_path[0]
if allow_access: # user allowed access - returns [true, fname0, fname1,...]
with MultipartWriter("images") as writer:
image_paths = image_path[1:]
for file_name in image_paths:
if os.path.isfile(file_name):
writer.append(open(file_name, "rb"))
else:
return web.Response(status=404)
return web.Response(body=writer, status=200)
else:
return web.Response(status=204) # that task already taken.
async def single_frame(request):
name = request.match_info['name']
response = web.StreamResponse(status=200,
reason='OK',
headers={
'Content-Type':
'multipart/x-mixed-replace; '
'boundary=--frame',
})
await response.prepare(request)
buffer = frameserver.get_buffer(name)
last = 0
while True:
with MultipartWriter('image/jpeg', boundary="frame") as mpwriter:
frame = buffer.get_latest_packet()
if frame is not None:
data = frame.to_image()
img_byte_arr = io.BytesIO()
data.save(img_byte_arr, format='JPEG')
img_byte_arr = img_byte_arr.getvalue()
mpwriter.append(img_byte_arr, {'Content-Type': 'image/jpeg'})
await mpwriter.write(response, False)
await asyncio.sleep(1 / 15)
await response.drain()
return response
def post_protobuf(self, url, protobuf_msg, data=None, **kwargs):
"""客户端POST请求protobuf版，会自动加上一些参数和头部
:param url: URL
:param protobuf_msg: protobuf message，一般是XXXReqIdl
:param data: body其他参数
:return: 同aiohttp的post
:raise aiohttp.ClientResponseError: HTTP请求失败
:raise aiohttp.ClientConnectionError: 连接失败
"""
data = {key: value if isinstance(value, str) else str(value)
for key, value in data.items()
} if data is not None else {}
self.__add_sign(data)
with MultipartWriter('form-data') as mp:
# MultipartWriter好像有BUG，append之后header就不能改了...
for key, value in data.items():
part = get_payload(str(value), headers={})
part.set_content_disposition('form-data', name=key)
mp.append_payload(part)
part = get_payload(protobuf_msg.SerializeToString(), headers={})
part.set_content_disposition('form-data', name='data', filename='file')
mp.append_payload(part)
return self._session.post(url, data=mp, headers={'x_bd_data_type': 'protobuf'},
**kwargs)
async def single(request):
id = int(request.match_info['id'])
response = web.StreamResponse(status=200,
reason='OK',
headers={
'Content-Type':
'multipart/x-mixed-replace; '
'boundary=--frame',
})
await response.prepare(request)
buffer = frameserver.get_buffer(id)
last = 0
while True:
with MultipartWriter('image/jpeg', boundary="frame") as mpwriter:
packet = buffer.get_latest_keyframe()
if packet is not None and packet.pts > last:
last = packet.pts
frame = packet.decode()
if len(frame) == 0:
continue
data = frame[0].to_image()
img_byte_arr = io.BytesIO()
data.save(img_byte_arr, format='JPEG')
img_byte_arr = img_byte_arr.getvalue()
mpwriter.append(img_byte_arr, {'Content-Type': 'image/jpeg'})
await mpwriter.write(response, False)
await asyncio.sleep(1 / 15)
await response.drain()
return response
async def mjpeg_handler(request):
"""This async function it's responsible
to create the MJPEG streaming.
Notes:
------
endpoint : /video/mjpeg
"""
my_boundary = 'image-boundary'
response = web.StreamResponse(
status=200,
reason='OK',
headers={
'Content-Type':
'multipart/x-mixed-replace;boundary={}'.format(my_boundary)
})
await response.prepare(request)
image_buffer_manager = request.app['image_buffer_manager']
while True:
jpeg_bytes = await image_buffer_manager.async_get_jpeg()
with MultipartWriter('image/jpeg', boundary=my_boundary) as mpwriter:
mpwriter.append(jpeg_bytes, {'Content-Type': 'image/jpeg'})
try:
await mpwriter.write(response, close_boundary=False)
except ConnectionResetError:
logging.info("Client connection closed")
break
await response.write(b"\r\n")
async def handle_mjpeg(request: Request):
logger.info(f"Client connected to MJPEG stream '{path}'")
boundary = 'frame-start'
response = web.StreamResponse(
status=200,
reason='OK',
headers={
"Content-Type":
f"multipart/x-mixed-replace;boundary={boundary}"
})
try:
await response.prepare(request)
while True:
stream.is_streaming = True
with MultipartWriter('image/jpeg',
boundary=boundary) as writer:
writer.append(stream.raw_image_data,
{'Content-Type': 'image/jpeg'})
await writer.write(response, close_boundary=False)
await asyncio.sleep(1 / stream_rate)
except Exception as e:
logger.info(
f"Client disconnected from MJPEG stream '{path}' because '{e}'"
)
stream.is_streaming = False
def MgetOriginalImages(self, data, request):
"""Return the non-graded original images for a task/question.
Respond with status 200/204.
Args:
data (dict): A dictionary having the user/token.
request (aiohttp.web_request.Request): Request of type
GET /MK/originalImages/`task code` which the task code
is extracted from.
Returns:
aiohttp.web_response.Response: A response object with includes the multipart objects
which wrap this task/question's original (ungraded) images.
"""
task = request.match_info["task"]
get_image_results = self.server.MgetOriginalImages(task)
image_return_success = get_image_results[0]
# returns either [True, fname1, fname2,... ] or [False]
if image_return_success:
original_image_paths = get_image_results[1:]
with MultipartWriter("images") as multipart_writer:
for file_nam in original_image_paths:
multipart_writer.append(open(file_nam, "rb"))
return web.Response(body=multipart_writer, status=200)
else:
return web.Response(status=204) # no content there
async def fire_request(request):
request.headers["Accept-Encoding"] = "gzip"
request.headers["User-Agent"] = "Aiogoogle Aiohttp (gzip)"
if request.media_upload:
# Validate
if (
request.media_upload.validate is True
and request.media_upload.max_size is not None
):
size = await _get_file_size(request.media_upload.file_path)
max_size = request.media_upload.max_size
if size > max_size:
raise ValidationError(
f'"{request.media_upload.file_path}" has a size of {size/1000}KB. Max upload size for this endpoint is: {max_size/1000}KB.'
)
# If multipart pass a file async generator
if request.media_upload.multipart is True:
with MultipartWriter('mixed') as mpwriter:
mpwriter.append(
_aiter_file(
request.media_upload.file_path,
request.media_upload.chunk_size
)
)
mpwriter.append_json(request.json)
return await self.request(
method=request.method,
url=request.media_upload.upload_path,
headers=request.headers,
data=mpwriter,
timeout=request.timeout,
verify_ssl=request._verify_ssl,
)
# Else load file to memory and send
else:
async with aiofiles.open(
request.media_upload.file_path, "rb"
) as file:
read_file = await file.read()
return await self.request(
method=request.method,
url=request.media_upload.upload_path,
headers=request.headers,
data=read_file,
json=request.json,
timeout=request.timeout,
verify_ssl=request._verify_ssl,
)
# Else, if no file upload
else:
return await self.request(
method=request.method,
url=request.url,
headers=request.headers,
data=request.data,
json=request.json,
timeout=request.timeout,
verify_ssl=request._verify_ssl,
)
def MgetWholePaper(self, data, request):
"""Return the entire paper which includes the given question.
Respond with status 200/404.
Args:
data (dict): A dictionary having the user/token.
request (aiohttp.web_request.Request): GET /MK/whole/`test_number`/`question_number`.
Returns:
aiohttp.web_response.Response: Responds with a multipart
writer which includes all the images for the exam which
includes this question.
"""
test_number = request.match_info["number"]
question_number = request.match_info["question"]
# return [True, pageData, f1, f2, f3, ...] or [False]
# 1. True/False for operation status.
# 2. A list of lists, documented elsewhere (TODO: I hope)
# 3. 3rd element onward: paths for each page of the paper in server.
whole_paper_response = self.server.MgetWholePaper(
test_number, question_number)
if not whole_paper_response[0]:
return web.Response(status=404)
with MultipartWriter("images") as multipart_writer:
pages_data = whole_paper_response[1]
all_pages_paths = whole_paper_response[2:]
multipart_writer.append_json(pages_data)
for file_name in all_pages_paths:
multipart_writer.append(open(file_name, "rb"))
return web.Response(body=multipart_writer, status=200)
async def write_new_parts(data, boundary, response):
with MultipartWriter("image/jpeg", boundary=boundary) as mpwriter:
mpwriter.append(data, {"Content-Type": "image/jpeg"})
# mpwriter.append(byte_im, {"Content-Type": "image/jpeg"})
await mpwriter.write(response, close_boundary=False)
logger.debug(f"Append response")
await response.drain()
async def write_to_stream(frame, response):
with MultipartWriter('image/jpeg', boundary='FRAME') as writer:
writer.append(frame, {
'Content-Type': 'image/jpeg',
'Content-Length': len(frame)
})
await writer.write(response, close_boundary=False)
def payload_params_to_multipart(params: Params["Payload"]) -> "MultipartWriter":
import json
from multidict import CIMultiDict
from aiohttp.multipart import MultipartWriter
multipart = MultipartWriter(subtype="form-data")
for key, payload in params.items():
multipart.append(
payload.data,
headers=CIMultiDict(
(
(PAYLOAD_META_HEADER, json.dumps(payload.meta)),
("Content-Disposition", f'form-data; name="{key}"'),
)
),
)
return multipart
async def write_stream_frame(self, response, frame) -> None:
"""Writes a frame to the camera stream.
:param response: Response
:param frame: Frame
"""
with MultipartWriter('image/jpeg', boundary='jpgboundary') as mpwriter:
mpwriter.append(frame, {'Content-Type': 'image/jpeg'})
await mpwriter.write(response, close_boundary=False)
await response.drain()
async def test_validator_with_default_config(http_service: Any) -> None:
"""Should return OK and successful validation."""
url = f"{http_service}/validator"
data_graph_file = "tests/files/valid_catalog.ttl"
shapes_graph_file = "tests/files/mock_dcat-ap-no-shacl_shapes_2.00.ttl"
ontology_graph_file = "tests/files/ontologies.ttl"
config = {"expand": True, "includeExpandedTriples": False}
with MultipartWriter("mixed") as mpwriter:
p = mpwriter.append(open(data_graph_file, "rb"))
p.set_content_disposition("attachment",
name="data-graph-file",
filename=data_graph_file)
p.headers[hdrs.CONTENT_ENCODING] = "gzip"
p = mpwriter.append(json.dumps(config))
p.set_content_disposition("inline", name="config")
p = mpwriter.append(open(shapes_graph_file, "rb"))
p.set_content_disposition("attachment",
name="shapes-graph-file",
filename=shapes_graph_file)
p = mpwriter.append(open(ontology_graph_file, "rb"))
p.set_content_disposition("attachment",
name="ontology-graph-file",
filename=ontology_graph_file)
session = ClientSession()
async with session.post(url, data=mpwriter) as resp:
body = await resp.text()
await session.close()
assert resp.status == 200
assert "text/turtle" in resp.headers[hdrs.CONTENT_TYPE]
# results_graph (validation report) should be isomorphic to the following:
src = """
@prefix sh: <http://www.w3.org/ns/shacl#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
[] a sh:ValidationReport ;
sh:conforms true
.
"""
with open(data_graph_file, "r") as file:
text = file.read()
g0 = Graph().parse(data=text, format="text/turtle")
g1 = g0 + Graph().parse(data=src, format="text/turtle")
g2 = Graph().parse(data=body, format="text/turtle")
_isomorphic = isomorphic(g1, g2)
if not _isomorphic:
_dump_diff(g1, g2)
pass
assert _isomorphic, "results_graph is incorrect"
async def fire_request(request):
request.headers["Accept-Encoding"] = "gzip"
request.headers["User-Agent"] = "Aiogoogle Aiohttp (gzip)"
if request.media_upload:
# Validate
await request.media_upload.run_validation(_get_file_size)
# If multipart pass a file async generator
if request.media_upload.multipart is True:
with MultipartWriter('mixed') as mpwriter:
mpwriter.append_json(request.json)
mpwriter.append(
request.media_upload.aiter_file(_aiter_file),
headers={"Content-Type": request.upload_file_content_type} if request.upload_file_content_type else None
)
req_content_type = 'multipart/related'
request.headers.update({"Content-Type": f"{req_content_type}; boundary={mpwriter.boundary}"})
return await self.request(
method=request.method,
url=request.media_upload.upload_path,
headers=request.headers,
data=mpwriter,
timeout=request.timeout,
verify_ssl=request._verify_ssl,
)
# Else load file to memory and send
else:
read_file = await request.media_upload.read_file(_read_file)
if request.upload_file_content_type:
request.headers.update({"Content-Type": request.upload_file_content_type})
return await self.request(
method=request.method,
url=request.media_upload.upload_path,
headers=request.headers,
data=read_file,
json=request.json,
timeout=request.timeout,
verify_ssl=request._verify_ssl,
)
# Else, if no file upload
else:
return await self.request(
method=request.method,
url=request.url,
headers=request.headers,
data=request.data,
json=request.json,
timeout=request.timeout,
verify_ssl=request._verify_ssl,
)
def MgetImages(self, data, request):
"""Return the image of a question/task to the client.
Main API call for the client to get the image data (original and annotated).
Respond with status 200/409.
Args:
data (dict): A dictionary having the user/token.
request (aiohttp.web_request.Request): Request of type GET /MK/images/"task code"
which the task code is extracted from.
Returns:
aiohttp.web_response.Response: A response which includes the multipart writer object
wrapping the task images.
"""
task_code = request.match_info["task"]
task_image_results = self.server.MgetImages(data["user"], task_code,
data["integrity_check"])
# A list which includes:
# 1. True/False process status.
# 2. Number of pages for task.
# 3. Original Page image path.
# 3. Marked Page image path.
# 4. Marked question plom data path, ie .plom type files.
# Format is either:
# [True, num_pages, original_fname1, original_fname2, ... original_fname#num_pages, ] or
# [True, num_pages, original_fname1,..,original_fname#num_pages, annotated_fname#1, ... annotated_fname#num_pages , plomdat] or
# [False, error]
task_image_success = task_image_results[0]
if task_image_success:
with MultipartWriter("imageAnImageAndPlom") as multipart_writer:
task_num_images = task_image_results[1]
task_images_list = task_image_results[2:]
multipart_writer.append(
"{}".format(task_num_images)) # send 'n' as string
for file_name in task_images_list:
multipart_writer.append(open(file_name, "rb"))
return web.Response(body=multipart_writer, status=200)
else:
if task_image_results[1] == "owner":
return web.Response(
status=409) # someone else has that task_image
elif task_image_results[1] == "integrity_fail":
return web.Response(status=406) # task changed
elif task_image_results[1] == "no_such_task":
return web.Response(status=410) # task deleted
else:
return web.Response(status=400) # some other error
def _encode_file(self, filename):
"""Encode file into multipart data.
:param filename: Full path to the javascript file or zip bundle.
:type filename: str | unicode
:return: Multipart encoder object
:rtype: aiohttp.MultipartWriter
"""
extension = os.path.splitext(filename)[1]
if extension == '.js': # pragma: no cover
source_type = 'application/javascript'
elif extension == '.zip':
source_type = 'application/zip'
else:
raise ValueError('File extension must be .zip or .js')
mpwriter = MultipartWriter()
mpwriter.append(
open(filename, 'rb').read(), {'CONTENT-TYPE': source_type})
return mpwriter
async def _process_multipart(reader: MultipartReader,
writer: MultipartWriter) -> None:
"""Process multipart.
Args:
reader (MultipartReader): Response multipart to process.
writer (MultipartWriter): Multipart to write out.
"""
while True:
part = await reader.next() # noqa: B305
# https://github.com/PyCQA/flake8-bugbear/issues/59
if part is None:
break
if isinstance(part, MultipartReader):
await _process_multipart(part, writer)
elif part.headers.get("hdrs.CONTENT_TYPE"):
if part.headers[hdrs.CONTENT_TYPE] == "application/json":
part_data: Optional[Union[Text, Dict[Text, Any],
List[Tuple[Text, Text]],
bytes]] = await part.json()
writer.append_json(part_data)
elif part.headers[hdrs.CONTENT_TYPE].startswith("text"):
part_data = await part.text()
writer.append(part_data)
elif part.headers[
hdrs.
CONTENT_TYPE] == "application/www-urlform-encode":
part_data = await part.form()
writer.append_form(part_data)
else:
part_data = await part.read()
writer.append(part_data)
else:
part_data = await part.read()
if part.name:
self.data.update({part.name: part_data})
elif part.filename:
part_data = await part.read()
self.data.update({part.filename: part_data})
writer.append(part_data)
async def mjpeg_handler(request):
print("mjpeg_handler=", request.url)
#browse(locals(), 'locals()')
#param1=request.get('client');
com = request.rel_url.query['com']
print("com=", com)
if com == 'start':
# param2 = request.rel_url.query['age']
# result = "name: {}, age: {}".format(param1, param2)
boundary = "boundarydonotcross"
responseImage = web.StreamResponse(status=200,
reason='OK',
headers={
'Content-Type':
'multipart/x-mixed-replace; '
'boundary=--%s' % boundary,
})
#responseImage.content_type = 'multipart/x-mixed-replace;boundary=ffserver'
await responseImage.prepare(
request
) #Send HTTP header. You should not change any header data after calling this method.
VC = cv2.VideoCapture(0)
request.app["camera"] = VC
encode_param = (int(cv2.IMWRITE_JPEG_QUALITY), 90)
request.app["streaming"].add(responseImage)
while True:
try:
_, frame = VC.read()
#await
if frame is None:
break
with MultipartWriter('image/jpeg',
boundary=boundary) as mpwriter:
result, encimg = cv2.imencode('.jpg', frame, encode_param)
data = encimg.tostring()
mpwriter.append(data, {'Content-Type': 'image/jpeg'})
await mpwriter.write(responseImage, close_boundary=False)
print("next frame")
await asyncio.sleep(0.010) #await responseImage.drain()
except asyncio.CancelledError as e:
request.app["streaming"].remove(responseImage)
request.app["camera"].remove(VC)
print("Exit camera mjpeg_handler")
VC.shutdown()
return responseImage
else:
print("streaming_clear", request.app["camera"])
if request.app["camera"] is not None:
request.app["camera"].release()
return web.Response()
async def edit_message(
self, channel_id: int, message_id: int, payload: dict, files: Optional[List[File]] = MISSING
) -> dict:
"""
Edits a message that already exists.
:param channel_id: Channel snowflake ID.
:param message_id: Message snowflake ID.
:param payload: Any new data that needs to be changed.
:param files: An optional list of files to send attached to the message.
:type payload: dict
:return: A message object with edited attributes.
"""
data = None
if files is not MISSING and len(files) > 0:
data = MultipartWriter("form-data")
part = data.append_json(payload)
part.set_content_disposition("form-data", name="payload_json")
payload = None
for id, file in enumerate(files):
part = data.append(
file._fp,
)
part.set_content_disposition(
"form-data", name="files[" + str(id) + "]", filename=file._filename
)
return await self._req.request(
Route(
"PATCH",
"/channels/{channel_id}/messages/{message_id}",
channel_id=channel_id,
message_id=message_id,
),
json=payload,
data=data,
)
async def test_validator_with_skos_ap_no(http_service: Any) -> None:
"""Should return OK and successful validation."""
url = f"{http_service}/validator"
data_graph_file = "tests/files/valid_collection.ttl"
shapes_graph_file = "tests/files/mock_skos-ap-no-shacl_shapes.ttl"
ontology_graph_file = "tests/files/skos_ontologies.ttl"
with MultipartWriter("mixed") as mpwriter:
p = mpwriter.append(open(data_graph_file, "rb"))
p.set_content_disposition("attachment",
name="data-graph-file",
filename=data_graph_file)
p = mpwriter.append(open(shapes_graph_file, "rb"))
p.set_content_disposition("attachment",
name="shapes-graph-file",
filename=shapes_graph_file)
p = mpwriter.append(open(ontology_graph_file, "rb"))
p.set_content_disposition("attachment",
name="ontology-graph-file",
filename=ontology_graph_file)
session = ClientSession()
async with session.post(url, data=mpwriter) as resp:
body = await resp.text()
await session.close()
assert resp.status == 200
assert "text/turtle" in resp.headers[hdrs.CONTENT_TYPE]
# results_graph (validation report) should be isomorphic to the following:
src = """
@prefix sh: <http://www.w3.org/ns/shacl#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
[] a sh:ValidationReport ;
sh:conforms true
.
"""
with open("tests/files/valid_collection.ttl", "r") as file:
text = file.read()
# body is graph of both the input data and the validation report
g0 = Graph().parse(data=text, format="text/turtle")
g1 = g0 + Graph().parse(data=src, format="turtle")
g2 = Graph().parse(data=body, format="text/turtle")
_isomorphic = isomorphic(g1, g2)
if not _isomorphic:
_dump_diff(g1, g2)
pass
assert _isomorphic, "results_graph is incorrect"
async def test_validator_url(http_service: Any) -> None:
"""Should return OK and successful validation."""
url = f"{http_service}/validator"
data_graph_url = "https://raw.githubusercontent.com/Informasjonsforvaltning/dcat-ap-no-validator-service/main/tests/files/valid_catalog.ttl" # noqa: B950
shapes_graph_file = "tests/files/mock_dcat-ap-no-shacl_shapes_2.00.ttl"
ontology_graph_file = "tests/files/ontologies.ttl"
with MultipartWriter("mixed") as mpwriter:
p = mpwriter.append(data_graph_url)
p.set_content_disposition("inline", name="data-graph-url")
p = mpwriter.append(open(shapes_graph_file, "rb"))
p.set_content_disposition("attachment",
name="shapes-graph-file",
filename=shapes_graph_file)
p = mpwriter.append(open(ontology_graph_file, "rb"))
p.set_content_disposition("attachment",
name="ontology-graph-file",
filename=ontology_graph_file)
session = ClientSession()
async with session.post(url, data=mpwriter) as resp:
body = await resp.text()
await session.close()
assert resp.status == 200
assert "text/turtle" in resp.headers[hdrs.CONTENT_TYPE]
# results_graph (validation report) should be isomorphic to the following:
src = """
@prefix sh: <http://www.w3.org/ns/shacl#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
[] a sh:ValidationReport ;
sh:conforms true
.
"""
session = ClientSession()
async with session.get(data_graph_url) as resp:
text = await resp.text()
await session.close()
g0 = Graph().parse(data=text, format="text/turtle")
g1 = g0 + Graph().parse(data=src, format="text/turtle")
g2 = Graph().parse(data=body, format="text/turtle")
_isomorphic = isomorphic(g1, g2)
if not _isomorphic:
_dump_diff(g1, g2)
pass
assert _isomorphic, "results_graph is incorrect"
async def create_message(
self, payload: dict, channel_id: int, files: Optional[List[File]] = MISSING
) -> dict:
"""
Send a message to the specified channel.
:param payload: Dictionary contents of a message. (i.e. message payload)
:param channel_id: Channel snowflake ID.
:param files: An optional list of files to send attached to the message.
:return dict: Dictionary representing a message (?)
"""
data = None
if files is not MISSING and len(files) > 0:
data = MultipartWriter("form-data")
part = data.append_json(payload)
part.set_content_disposition("form-data", name="payload_json")
payload = None
for id, file in enumerate(files):
part = data.append(
file._fp,
)
part.set_content_disposition(
"form-data", name="files[" + str(id) + "]", filename=file._filename
)
request = await self._req.request(
Route("POST", "/channels/{channel_id}/messages", channel_id=channel_id),
json=payload,
data=data,
)
if request.get("id"):
self.cache.messages.add(Item(id=request["id"], value=Message(**request)))
return request
