async def test_set_content_disposition_override(self, buf: Any,
stream: Any) -> None:
# https://github.com/aio-libs/aiohttp/pull/3475#issuecomment-451072381
with pathlib.Path(__file__).open("rb") as fobj:
with aiohttp.MultipartWriter("form-data", boundary=":") as writer:
part = writer.append(
fobj,
headers={
CONTENT_DISPOSITION: 'attachments; filename="bug.py"',
CONTENT_TYPE: "text/python",
},
)
content_length = part.size
await writer.write(stream)
assert part.headers[CONTENT_TYPE] == "text/python"
assert part.headers[CONTENT_DISPOSITION] == (
'attachments; filename="bug.py"')
headers, _ = bytes(buf).split(b"\r\n\r\n", 1)
assert headers == (
b"--:\r\n"
b"Content-Type: text/python\r\n"
b'Content-Disposition: attachments; filename="bug.py"\r\n'
b"Content-Length: %s"
b"" % (str(content_length).encode(), ))
async def upload_file(self):
""" Upload the tarfile to cloud as multipart data"""
logger.debug("uploading %s", self.tgzfile)
with aiohttp.MultipartWriter("form-data") as mpwriter:
with open(self.tgzfile, "rb") as file_handle:
part = mpwriter.append(file_handle)
part.set_content_disposition("form-data",
name="file",
filename="inventory.gz")
part.headers[
aiohttp.hdrs.CONTENT_TYPE] = self.UPLOAD_CONTENT_TYPE
headers = {}
# TODO : Use mTLS certs not userid/password
auth = aiohttp.BasicAuth(self.config["AUTH"]["username"],
self.config["AUTH"]["password"])
headers["Authorization"] = auth.encode()
async with aiohttp.ClientSession(headers=headers) as session:
async with session.post(self.upload_url,
ssl=self.ssl_context,
data=mpwriter) as response:
logger.debug("Status: %s", response.status)
logger.debug("Content-type: %s",
response.headers["Content-Type"])
return await response.text()
def test_append_multipart(self, writer: Any) -> None:
subwriter = aiohttp.MultipartWriter(boundary=":")
subwriter.append_json({"foo": "bar"})
writer.append(subwriter, {CONTENT_TYPE: "test/passed"})
assert 1 == len(writer)
part = writer._parts[0][0]
assert part.headers[CONTENT_TYPE] == "test/passed"
async def test_reset_content_disposition_header(self, buf, stream):
"""
https://github.com/aio-libs/aiohttp/pull/3475#issuecomment-451072381
"""
with open(__file__, 'rb') as fobj:
with aiohttp.MultipartWriter('form-data', boundary=':') as writer:
part = writer.append(
fobj,
headers={CONTENT_TYPE: 'text/plain'},
)
content_length = part.size
assert CONTENT_DISPOSITION in part.headers
part.set_content_disposition('attachments', filename='bug.py')
await writer.write(stream)
headers, _ = bytes(buf).split(b'\r\n\r\n', 1)
assert headers == (
b'--:\r\n'
b'Content-Type: text/plain\r\n'
b'Content-Disposition:'
b' attachments; filename="bug.py"; filename*=utf-8\'\'bug.py\r\n'
b'Content-Length: %s'
b'' % (str(content_length).encode(), ))
async def download_files(request) -> web.Response:
try:
registry = request.app['registry']
sess_id = request.match_info['sess_id']
access_key = request['keypair']['access_key']
with _timeout(2):
params = await request.json(loads=json.loads)
assert params.get('files'), 'no file(s) specified!'
files = params.get('files')
log.info('DOWNLOAD_FILE (u:{0}, token:{1})', access_key, sess_id)
except (asyncio.TimeoutError, AssertionError,
json.decoder.JSONDecodeError) as e:
log.warning('DOWNLOAD_FILE: invalid/missing parameters, {0!r}', e)
raise InvalidAPIParameters(extra_msg=str(e.args[0]))
try:
assert len(files) <= 5, 'Too many files'
await registry.increment_session_usage(sess_id, access_key)
# TODO: Read all download file contents. Need to fix by using chuncking, etc.
results = await asyncio.gather(*map(
functools.partial(registry.download_file, sess_id, access_key),
files))
log.debug('file(s) inside container retrieved')
except BackendError:
log.exception('DOWNLOAD_FILE: exception')
raise
except Exception:
request.app['error_monitor'].capture_exception()
log.exception('DOWNLOAD_FILE: unexpected error!')
raise InternalServerError
with aiohttp.MultipartWriter('mixed') as mpwriter:
for tarbytes in results:
mpwriter.append(tarbytes)
return web.Response(body=mpwriter, status=200)
async def download(request, row):
folder_name = request.match_info['name']
access_key = request['keypair']['access_key']
params = await request.json()
assert params.get('files'), 'no file(s) specified!'
files = params.get('files')
log.info('VFOLDER.DOWNLOAD (u:{0}, f:{1})', access_key, folder_name)
folder_path = (request.app['VFOLDER_MOUNT'] / row.host / row.id.hex)
for file in files:
if not (folder_path / file).is_file():
raise InvalidAPIParameters(
f'You cannot download "{file}" because it is not a regular file.'
)
with aiohttp.MultipartWriter('mixed') as mpwriter:
total_payloads_length = 0
headers = {'Content-Encoding': 'gzip'}
try:
for file in files:
data = open(folder_path / file, 'rb')
payload = mpwriter.append(data, headers)
total_payloads_length += payload.size
except FileNotFoundError:
return web.Response(status=404, reason='File not found')
mpwriter._headers['X-TOTAL-PAYLOADS-LENGTH'] = str(
total_payloads_length)
return web.Response(body=mpwriter, status=200)
async def test_reset_content_disposition_header(self, buf: Any,
stream: Any) -> None:
# https://github.com/aio-libs/aiohttp/pull/3475#issuecomment-451072381
with pathlib.Path(__file__).open("rb") as fobj:
with aiohttp.MultipartWriter("form-data", boundary=":") as writer:
part = writer.append(
fobj,
headers={CONTENT_TYPE: "text/plain"},
)
content_length = part.size
assert CONTENT_DISPOSITION in part.headers
part.set_content_disposition("attachments", filename="bug.py")
await writer.write(stream)
headers, _ = bytes(buf).split(b"\r\n\r\n", 1)
assert headers == (
b"--:\r\n"
b"Content-Type: text/plain\r\n"
b"Content-Disposition:"
b" attachments; filename=\"bug.py\"; filename*=utf-8''bug.py\r\n"
b"Content-Length: %s"
b"" % (str(content_length).encode(), ))
async def _perform_async_post(self,
content: str,
syntax: str = None) -> Paste:
""" Async post request. """
if not self.session and self._are_we_async:
self.session = await self._generate_async_session()
multi_part_write = aiohttp.MultipartWriter()
paste_content = multi_part_write.append(content)
paste_content.set_content_disposition("form-data", name="data")
paste_content = multi_part_write.append_json(
{"meta": [{
"index": 0,
"syntax": syntax
}]})
paste_content.set_content_disposition("form-data", name="meta")
async with self.session.post(API_BASE_URL,
data=multi_part_write) as response:
status_code = response.status
response_text = await response.text()
if status_code not in (200, ):
raise APIError(status_code, response_text)
response_data = await response.json()
return Paste(response_data, syntax)
async def test_writer_write_no_close_boundary(buf, stream) -> None:
writer = aiohttp.MultipartWriter(boundary=':')
writer.append('foo-bar-baz')
writer.append_json({'test': 'passed'})
writer.append_form({'test': 'passed'})
writer.append_form([('one', 1), ('two', 2)])
await writer.write(stream, close_boundary=False)
assert (
(b'--:\r\n'
b'Content-Type: text/plain; charset=utf-8\r\n'
b'Content-Length: 11\r\n\r\n'
b'foo-bar-baz'
b'\r\n'
b'--:\r\n'
b'Content-Type: application/json\r\n'
b'Content-Length: 18\r\n\r\n'
b'{"test": "passed"}'
b'\r\n'
b'--:\r\n'
b'Content-Type: application/x-www-form-urlencoded\r\n'
b'Content-Length: 11\r\n\r\n'
b'test=passed'
b'\r\n'
b'--:\r\n'
b'Content-Type: application/x-www-form-urlencoded\r\n'
b'Content-Length: 11\r\n\r\n'
b'one=1&two=2'
b'\r\n') == bytes(buf))
def _make_multipart(self):
"""
构造multipart/form-data 类型的aiohttp请求参数
Returns:
aiohttp.MultipartWriter实例
"""
mpwriter = aiohttp.MultipartWriter('form-data')
post_data = self.http_data["data"]
for key in post_data:
part = mpwriter.append(post_data[key])
part.set_content_disposition("form-data", name=key)
part.headers.pop(aiohttp.hdrs.CONTENT_LENGTH, None)
part.headers.pop(aiohttp.hdrs.CONTENT_TYPE, None)
files = self.http_data["files"]
for file_item in files:
content_type = file_item.get("content_type",
'application/octet-stream')
part = mpwriter.append(file_item["content"],
{'CONTENT-TYPE': content_type})
part.set_content_disposition("form-data",
filename=file_item["filename"],
name=file_item["name"])
part.headers.pop(aiohttp.hdrs.CONTENT_LENGTH, None)
Logger().debug("Make multipart data from dict: {}".format(post_data))
return mpwriter
async def test_set_content_disposition_override(self, buf, stream):
"""
https://github.com/aio-libs/aiohttp/pull/3475#issuecomment-451072381
"""
with open(__file__, 'rb') as fobj:
with aiohttp.MultipartWriter('form-data', boundary=':') as writer:
part = writer.append(fobj,
headers={
CONTENT_DISPOSITION:
'attachments; filename="bug.py"',
CONTENT_TYPE: 'text/python',
})
content_length = part.size
await writer.write(stream)
assert part.headers[CONTENT_TYPE] == 'text/python'
assert part.headers[CONTENT_DISPOSITION] == (
'attachments; filename="bug.py"')
headers, _ = bytes(buf).split(b'\r\n\r\n', 1)
assert headers == (
b'--:\r\n'
b'Content-Type: text/python\r\n'
b'Content-Disposition: attachments; filename="bug.py"\r\n'
b'Content-Length: %s'
b'' % (str(content_length).encode(), ))
def test_append_multipart(self, writer) -> None:
subwriter = aiohttp.MultipartWriter(boundary=':')
subwriter.append_json({'foo': 'bar'})
writer.append(subwriter, {CONTENT_TYPE: 'test/passed'})
assert 1 == len(writer)
part = writer._parts[0][0]
assert part.headers[CONTENT_TYPE] == 'test/passed'
async def test_writer_write_no_close_boundary(buf: Any, stream: Any) -> None:
writer = aiohttp.MultipartWriter(boundary=":")
writer.append("foo-bar-baz")
writer.append_json({"test": "passed"})
writer.append_form({"test": "passed"})
writer.append_form([("one", 1), ("two", 2)])
await writer.write(stream, close_boundary=False)
assert (b"--:\r\n"
b"Content-Type: text/plain; charset=utf-8\r\n"
b"Content-Length: 11\r\n\r\n"
b"foo-bar-baz"
b"\r\n"
b"--:\r\n"
b"Content-Type: application/json\r\n"
b"Content-Length: 18\r\n\r\n"
b'{"test": "passed"}'
b"\r\n"
b"--:\r\n"
b"Content-Type: application/x-www-form-urlencoded\r\n"
b"Content-Length: 11\r\n\r\n"
b"test=passed"
b"\r\n"
b"--:\r\n"
b"Content-Type: application/x-www-form-urlencoded\r\n"
b"Content-Length: 11\r\n\r\n"
b"one=1&two=2"
b"\r\n") == bytes(buf)
async def scan_file_async(self, file, wait_for_completion=False):
"""Like :func:`scan_file` but returns a coroutine."""
# The snippet below could be replaced with this simpler code:
#
# form_data = aiohttp.FormData()
# form_data.add_field('file', file)
#
# However, aiohttp.FormData assumes that the server supports RFC 5987 and
# send a Content-Disposition like:
#
# 'form-data; name="file"; filename="foobar"; filename*=UTF-8''foobar
#
# AppEngine's upload handler doesn't like the filename*=UTF-8''foobar field
# and fails with this Content-Disposition header.
part = aiohttp.get_payload(file)
filename = file.name if hasattr(file, 'name') else 'unknown'
disposition = 'form-data; name="file"; filename="{}"'.format(filename)
part.headers['Content-Disposition'] = disposition
form_data = aiohttp.MultipartWriter('form-data')
form_data.append_payload(part)
upload_url = await self.get_data_async('/files/upload_url')
response = ClientResponse(
await self._get_session().post(upload_url, data=form_data))
analysis = await self._response_to_object(response)
if wait_for_completion:
analysis = await self._wait_for_analysis_completion(analysis)
return analysis
async def test_training_too_many_fail(testrequest):
# we only have one training slot
for ii in range(1):
dev_id = 'devpost{}'.format(ii)
ai_id = 'aipost{}'.format(ii)
await test_status_update_1(testrequest, dev_id, ai_id)
dev_id = 'devpost2'.format(ii)
ai_id = 'aipost2'.format(ii)
cli = testrequest.cli
with aiohttp.MultipartWriter('training') as mpwriter:
mpwriter.append_json({'dev_id': dev_id, 'ai_id': ai_id})
payload = aiohttp.payload.TextIOPayload(open(TRAINING_FILE_PATH))
payload.set_content_disposition('attachment', filename='training.txt')
mpwriter.append_payload(payload)
async with cli.post('/ai', data=mpwriter) as resp:
assert resp.status == 200
json_data = await resp.json()
assert json_data['status'] == 'ai_ready_to_train'
assert 'url' in json_data
# check data is written in correctly
assert testrequest.mock_training.training_list[(
dev_id, ai_id)].status.state == ait.AiTrainingState.ai_ready_to_train
async with cli.post('/ai/{}/{}?command=start'.format(dev_id,
ai_id)) as resp:
# !!! THIS ONE SHOULD FAIL AS THERE ARE ALREADY AN AIs in TRAINING !!!
assert resp.status == 429
async def generate_tempfile(session, sessionId, sheetdocId):
with aiohttp.MultipartWriter('form-data') as mp:
data = {
'sheetdocId': sheetdocId,
'useTabs': 'true',
'sendNotifications': 'true',
}
for key, value in data.items():
part = mp.append(value)
part.set_content_disposition('form-data', name=key)
async with session.post(
f'https://visualizedata.ucop.edu/vizql/t/Public/w/TransferbyCCM/v/ByMajorName/sessions/{sessionId}/commands/tabsrv/export-crosstab-to-csvserver',
data=mp) as response:
if response.status != 200:
print(response)
print(await response.text())
raise RuntimeError
data = await response.json(content_type=None)
tempfileKey = data['vqlCmdResponse']['layoutStatus'][
'applicationPresModel']['presentationLayerNotification'][0][
'presModelHolder']['genFileDownloadPresModel'][
'tempfileKey']
return tempfileKey
async def asend(self, *, sess=None, timeout=10.0):
'''
Sends the request to the server.
This method is a coroutine.
'''
assert self.method in self._allowed_methods
if sess is None:
sess = aiohttp.ClientSession()
else:
assert isinstance(sess, aiohttp.ClientSession)
with sess:
if self.content_type == 'multipart/form-data':
with aiohttp.MultipartWriter('mixed') as mpwriter:
for file in self._content:
part = mpwriter.append(file.file)
part.set_content_disposition('attachment',
filename=file.filename)
data = mpwriter
else:
data = self._content
self._sign()
reqfunc = getattr(sess, self.method.lower())
try:
with _timeout(timeout):
resp = await reqfunc(self.build_url(),
data=data,
headers=self.headers)
async with resp:
body = await resp.read()
return Response(resp.status, resp.reason, body,
resp.content_type, len(body))
except Exception as e:
msg = 'Request to the API endpoint has failed.'
raise BackendClientError(msg) from e
async def call_multipart_mock_event_plain_text_json(client):
os.makedirs('/tmp/call_multipart_mock_event/', exist_ok=True)
with open('/tmp/call_multipart_mock_event/test_attachment', 'wb') as f:
f.write(b'testdata')
attachment = open('/tmp/call_multipart_mock_event/test_attachment', 'rb')
with aiohttp.MultipartWriter("form-data", boundary=":") as mp:
mp.append("value1",
headers={'Content-Disposition': 'form-data; name="field1"'})
mp.append('{"value": "value2"}',
headers={'Content-Disposition': 'form-data; name="field2"'})
mp.append(
attachment,
headers={
'Content-Disposition':
'attachments; name="attachment"; filename="test_attachment"'
})
res: ClientResponse = await client.post(
'/api/mock-app/test/mock-multipart-event-test',
params={'query_arg1': 'ok'},
data=mp,
headers={
'X-Track-Request-Id': 'test_request_id',
'X-Track-Session-Id': 'test_session_id',
'Content-Type': 'multipart/form-data; boundary=":"'
})
assert res.status == 200
assert res.headers.get('X-Track-Session-Id') == 'test_session_id'
assert res.headers.get('X-Track-Request-Id') == 'test_request_id'
result = (await res.read()).decode()
assert result == '{"value": "field1=value1 field2=value2 attachment=test_attachment ok"}'
async def test_upload_training_missing_data_1(cli):
dev_id = 'devpost1'
ai_id = 'aipost1'
with aiohttp.MultipartWriter('training') as mpwriter:
mpwriter.append_json({'dev_id': dev_id, 'ai_id': ai_id})
resp = await cli.post('/ai', data=mpwriter)
assert resp.status == 400
async def test_upload_training_bad_dev_id(cli):
with aiohttp.MultipartWriter('training') as mpwriter:
mpwriter.append_json({'ai_id': 'aipost1'})
payload = aiohttp.payload.TextIOPayload(open(TRAINING_FILE_PATH))
payload.set_content_disposition('attachment', filename='training.txt')
mpwriter.append_payload(payload)
resp = await cli.post('/ai', data=mpwriter)
assert resp.status == 400
async def make_mystbin(session, text):
wr = aiohttp.MultipartWriter()
t = wr.append(text)
t.set_content_disposition("form-data", name="data")
t = wr.append_json({"meta": [{"index": 0, "syntax": "python"}]})
t.set_content_disposition("form-data", name="meta")
async with session.post("https://mystb.in/api/pastes", data=wr) as resp:
return "https://mystb.in/" + (await
resp.json())["pastes"][0]["id"] + ".py"
async def request_http(data):
async with aiohttp.ClientSession() as session:
with aiohttp.MultipartWriter() as mpwriter:
part = mpwriter.append(data)
# part.set_content_disposition('binary')
part.headers[aiohttp.hdrs.CONTENT_TYPE] = 'binary'
mpwriter.append("the local size is " + str(len(data)))
async with session.post(full_url, data=mpwriter) as resp:
print('HTTP code is ', resp.status)
print(await resp.text())
def req1():
with aiohttp.MultipartWriter() as writer:
writer.append('test')
writer.append_json({'passed': True})
resp = yield from aiohttp.request("post",
'http://localhost:8080/multipart',
data=writer,
headers=writer.headers)
print(resp)
assert 200 == resp.status
async def req1():
with aiohttp.MultipartWriter() as writer:
writer.append('test')
writer.append_json({'passed': True})
resp = await aiohttp.ClientSession().request("post",
'http://localhost:8080/',
data=writer,
headers=writer.headers)
print(resp)
assert 200 == resp.status
async def test_upload_training_exists(cli, mock_training):
# try and upload to the one AI we know is already there
dev_id = 'd1'
ai_id = 'a1'
with aiohttp.MultipartWriter('training') as mpwriter:
mpwriter.append_json({'dev_id': dev_id, 'ai_id': ai_id})
payload = aiohttp.payload.TextIOPayload(open(TRAINING_FILE_PATH))
payload.set_content_disposition('attachment', filename='training.txt')
mpwriter.append_payload(payload)
resp = await cli.post('/ai', data=mpwriter)
assert resp.status == 200
await resp.json()
async def post(content: str, *, session: aiohttp.ClientSession = None, suffix: str = None) -> str:
""" Post `content` to MystB.in with optional suffix text. """
suffix = f".{suffix}" if suffix else ""
multi_part = aiohttp.MultipartWriter()
paste_content = multi_part.append(content)
paste_content.set_content_disposition("form-data", name="data")
paste_content = multi_part.append_json({"meta": [{"index": 0, "syntax": suffix}]})
paste_content.set_content_disposition("form-data", name="meta")
session = session or aiohttp.ClientSession(raise_for_status=True)
async with session.post(BASE, data=multi_part, timeout=TIMEOUT) as mb_res:
url = await mb_res.json()
short_id = url['pastes'][0]['id']
return f"https://mystb.in/{short_id}{suffix}"
async def req2():
with aiohttp.MultipartWriter() as writer:
writer.append('test')
writer.append_json({'passed': True})
writer.append(open('src/main.rs'))
resp = await aiohttp.ClientSession().request(
"post",
'http://localhost:7070/images',
data=writer,
headers=writer.headers)
print(resp)
assert 200 == resp.status
async def get_mystbin_link(bot: Bot, content: str, syntax: str = None):
multi_part_writer = aiohttp.MultipartWriter()
paste_content = multi_part_writer.append(content)
paste_content.set_content_disposition("form-data", name="data")
paste_content = multi_part_writer.append_json(
{"meta": [{
"index": 0,
"syntax": syntax
}]})
paste_content.set_content_disposition("form-data", name="meta")
async with bot.http_session.post(MYSTBIN_API_URL,
data=multi_part_writer) as response:
return await response.json()
def _prepare_file(self, content, filename):
with aiohttp.MultipartWriter('form-data') as data:
filesdata = aiohttp.payload.get_payload(content)
filesdata.set_content_disposition(
'form-data', name='files', filename=filename)
data.append(filesdata)
jsondata = {'submitter': self.submitter}
if self.submitter_id:
jsondata["submitter_id"] = self.submitter_id
jsondata = aiohttp.JsonPayload(jsondata)
jsondata.set_content_disposition('form-data', name='json')
data.append(jsondata)
return data
async def upload_data(self, data: bytes, token: str, key: str = None,
params: dict = None, filename: str = None,
mimetype: str = None, host: str = None) -> dict:
"""直传文件数据到七牛云
:param data: 上传的字节码数据
:param token: 上传凭证
:param key: 上传后的文件命名
:param params: 用户自定义参数，可为空，dict类型
:param filename: 上传的数据的文件名，默认为空
:param mimetype: 上传数据的mimetype值，默认为空，可由七牛自动探测
:param host: 上传的服务器地址，默认为"upload.qiniu.com"
:return: 上传后的文件信息，包含hash和key
详见：https://developer.qiniu.com/kodo/api/1312/upload
"""
filename = filename or b32encode(os.urandom(5)).decode()
params = params or {}
host = host or "http://upload.qiniu.com"
if host.startswith("http://") or host.startswith("https://"):
url = host
else:
url = "http://{}".format(host)
with aiohttp.MultipartWriter("form-data") as mpwriter:
mpwriter.append(token, {
"Content-Disposition": 'form-data; name="token"',
})
mpwriter.append(key, {
"Content-Disposition": 'form-data; name="key"',
})
for name, value in params.items():
mpwriter.append(value, {
"Content-Disposition": 'form-data; name="{}"'.format(name),
})
mpheaders = {
"Content-Disposition":
'form-data; name="file"; filename="{}"'.format(filename),
"Content-Transfer-Encoding": "binary",
}
if mimetype:
mpheaders["Content-Type"] = mimetype
mpwriter.append(BytesIO(data), mpheaders)
async with self.httpclient.post(url, data=mpwriter) as resp:
await raise_for_error(resp)
ret = await resp.json()
return ret
