async def _copy_file_multi_part_main(self, sema: asyncio.Semaphore,
source_report: SourceReport,
srcfile: str, srcstat: FileStatus,
destfile: str,
return_exceptions: bool):
size = await srcstat.size()
if size <= self.PART_SIZE:
await retry_transient_errors(self._copy_file, srcfile, destfile)
return
n_parts = int((size + self.PART_SIZE - 1) / self.PART_SIZE)
try:
part_creator = await self.router_fs.multi_part_create(
sema, destfile, n_parts)
except FileNotFoundError:
await self.router_fs.makedirs(os.path.dirname(destfile),
exist_ok=True)
part_creator = await self.router_fs.multi_part_create(
sema, destfile, n_parts)
async with part_creator:
await bounded_gather2(sema,
*[
retry_transient_errors(
self._copy_part, source_report,
srcfile, i, part_creator,
return_exceptions)
for i in range(n_parts)
],
cancel_on_error=True)
async def handle_ws_response(request, userdata, endpoint, f):
app = request.app
user_queries: Dict[str, asyncio.Future] = request.app['queries'][
userdata['username']]
ws = web.WebSocketResponse(heartbeat=30, max_msg_size=0)
await ws.prepare(request)
body = await ws.receive_json()
query = user_queries.get(body['token'])
if query is None:
await add_user(app, userdata)
query = asyncio.ensure_future(
retry_transient_errors(blocking_to_async, app['thread_pool'], f,
userdata, body))
user_queries[body['token']] = query
try:
receive = asyncio.ensure_future(ws.receive_str(
)) # receive automatically ping-pongs which keeps the socket alive
await asyncio.wait([receive, query],
return_when=asyncio.FIRST_COMPLETED)
if receive.done():
# we expect no messages from the client
response = receive.result()
raise AssertionError(
f'{endpoint}: client broke the protocol by sending: {response}'
)
if not query.done():
return
if query.exception() is not None:
exc = query.exception()
exc_str = traceback.format_exception(type(exc), exc,
exc.__traceback__)
await ws.send_json({'status': 500, 'value': exc_str})
else:
await ws.send_json({'status': 200, 'value': query.result()})
assert (await receive) == 'bye'
del user_queries[body['token']]
finally:
receive.cancel()
query.cancel()
await ws.close()
return ws
async def test_multi_part_create(filesystem: Tuple[asyncio.Semaphore, AsyncFS,
str], permutation):
sema, fs, base = filesystem
# S3 has a minimum part size (except for the last part) of 5MiB
if base.startswith('s3'):
min_part_size = 5 * 1024 * 1024
part_data_size = [min_part_size, min_part_size, min_part_size]
else:
part_data_size = [8192, 600, 20000]
part_data = [secrets.token_bytes(s) for s in part_data_size]
s = 0
part_start = []
for b in part_data:
part_start.append(s)
s += len(b)
path = f'{base}a'
async with await fs.multi_part_create(sema, path, len(part_data)) as c:
async def create_part(i):
async with await c.create_part(i, part_start[i]) as f:
await f.write(part_data[i])
if permutation:
# do it in a fixed order
for i in permutation:
await retry_transient_errors(create_part, i)
else:
# do in parallel
await asyncio.gather(*[
retry_transient_errors(create_part, i)
for i in range(len(part_data))
])
expected = b''.join(part_data)
async with await fs.open(path) as f:
actual = await f.read()
assert expected == actual
def request(self, endpoint, **data):
return async_to_blocking(retry_transient_errors(self.async_request, endpoint, **data))