async def _copy_file_multi_part_main(self, sema: asyncio.Semaphore, source_report: SourceReport, srcfile: str, srcstat: FileStatus, destfile: str, return_exceptions: bool): size = await srcstat.size() if size <= self.PART_SIZE: await retry_transient_errors(self._copy_file, srcfile, destfile) return n_parts = int((size + self.PART_SIZE - 1) / self.PART_SIZE) try: part_creator = await self.router_fs.multi_part_create( sema, destfile, n_parts) except FileNotFoundError: await self.router_fs.makedirs(os.path.dirname(destfile), exist_ok=True) part_creator = await self.router_fs.multi_part_create( sema, destfile, n_parts) async with part_creator: await bounded_gather2(sema, *[ retry_transient_errors( self._copy_part, source_report, srcfile, i, part_creator, return_exceptions) for i in range(n_parts) ], cancel_on_error=True)
async def handle_ws_response(request, userdata, endpoint, f): app = request.app user_queries: Dict[str, asyncio.Future] = request.app['queries'][ userdata['username']] ws = web.WebSocketResponse(heartbeat=30, max_msg_size=0) await ws.prepare(request) body = await ws.receive_json() query = user_queries.get(body['token']) if query is None: await add_user(app, userdata) query = asyncio.ensure_future( retry_transient_errors(blocking_to_async, app['thread_pool'], f, userdata, body)) user_queries[body['token']] = query try: receive = asyncio.ensure_future(ws.receive_str( )) # receive automatically ping-pongs which keeps the socket alive await asyncio.wait([receive, query], return_when=asyncio.FIRST_COMPLETED) if receive.done(): # we expect no messages from the client response = receive.result() raise AssertionError( f'{endpoint}: client broke the protocol by sending: {response}' ) if not query.done(): return if query.exception() is not None: exc = query.exception() exc_str = traceback.format_exception(type(exc), exc, exc.__traceback__) await ws.send_json({'status': 500, 'value': exc_str}) else: await ws.send_json({'status': 200, 'value': query.result()}) assert (await receive) == 'bye' del user_queries[body['token']] finally: receive.cancel() query.cancel() await ws.close() return ws
async def test_multi_part_create(filesystem: Tuple[asyncio.Semaphore, AsyncFS, str], permutation): sema, fs, base = filesystem # S3 has a minimum part size (except for the last part) of 5MiB if base.startswith('s3'): min_part_size = 5 * 1024 * 1024 part_data_size = [min_part_size, min_part_size, min_part_size] else: part_data_size = [8192, 600, 20000] part_data = [secrets.token_bytes(s) for s in part_data_size] s = 0 part_start = [] for b in part_data: part_start.append(s) s += len(b) path = f'{base}a' async with await fs.multi_part_create(sema, path, len(part_data)) as c: async def create_part(i): async with await c.create_part(i, part_start[i]) as f: await f.write(part_data[i]) if permutation: # do it in a fixed order for i in permutation: await retry_transient_errors(create_part, i) else: # do in parallel await asyncio.gather(*[ retry_transient_errors(create_part, i) for i in range(len(part_data)) ]) expected = b''.join(part_data) async with await fs.open(path) as f: actual = await f.read() assert expected == actual
def request(self, endpoint, **data): return async_to_blocking(retry_transient_errors(self.async_request, endpoint, **data))