async def read_file(file_path: str, background_tasks: BackgroundTasks, stats: bool = False): try: tasks = [] start = time.time() for i in range(settings.primary + settings.parity): tasks.append(receive_file_block(i, file_path)) pieces = await asyncio.gather(*tasks) transfer_time = time.time() - start start = time.time() file, piece_map = decode_data(pieces) temp_file = tempfile.NamedTemporaryFile(delete=False) async with aiofile.async_open(temp_file.name, 'wb') as f: await f.write(file.buffer) decode_time = time.time() - start background_tasks.add_task(delete_temp_file, temp_file.name) background_tasks.add_task(rebuild_redundancy, file, piece_map) if stats: return { 'delay': [transfer_time, decode_time] } else: return FileResponse(temp_file.name, media_type='application/octet-stream') except Exception as e: logger.exception(e) raise HTTPException(400, str(e))
async def send_file_block(server_id, file_path, piece): try: buffer = pickle.dumps(piece) if server_id == settings.server_id: await write_file_block_in_fs(file_path, buffer) logger.info('%s: sendblock %d to server %d', file_path, piece.piece_id, server_id) return True else: port = settings.base_port + server_id url = 'http://%s:%d/writeblock/%s' % (settings.host, port, file_path) headers = { 'content-type': 'application/octet-stream' } session = get_session() async with session.post(url, data=buffer, headers=headers) as resp: resp: aiohttp.ClientResponse if resp.status == 200: logger.info('%s: sendblock %d to server %d', file_path, piece.piece_id, server_id) return True else: raise Exception(resp) except Exception as e: logger.error('%s failed: sendblock %d to server %d', file_path, piece.piece_id, server_id) logger.exception(e) return False
async def authenticate(self, socket_message: SocketJsonInputMessage): """Check if user is valid""" client_id = socket_message.client_id token = socket_message.access_token processor_options = socket_message.data # Try one token for all if COMMON_TOKEN and token == COMMON_TOKEN: self.is_authenticated = True # Try user list elif client_id and token: if client_id in settings.user_tokens: user_token = settings.user_tokens[client_id] if user_token is not None and user_token == token: self.is_authenticated = True # Create processor if self.is_authenticated: try: self.processor = ChunkProcessor(processor_name=None, send_message=self.send_message, options=processor_options) except RuntimeError: logger.exception("ChunkProcessor - Failed to create processor") await self.send_message(SocketErrorMessage(500, "ChunkProcessorError", "Failed to create processor.")) else: logger.warning("User %s failed to authenticate!", client_id) await asyncio.sleep(3)
async def process(self, chunk: bytes): """Write chunks to file""" try: self._file.write(chunk) except OSError: logger.exception("WaveFileWriter - Failed to process") self.on_error( "Engine: wave_file_writer - Message: Failed to process")
async def delete_file_block(file_path: str, timestamp: int): try: async with write_file_block_lock: result = await delete_file_block_in_fs(file_path, timestamp) return { 'result': result } except Exception as e: logger.exception(e) raise HTTPException(400, str(e))
def _close_file(self): """Try to close file""" try: if self._file is not None and not self._file.closed: self._file.close() logger.info("WaveFileWriter - File closed: %s", self._file_name) except OSError: logger.exception("WaveFileWriter - Failed to close file") self.on_error( "Engine: wave_file_writer - Message: Failed to close")
async def read_file_block(file_path: str): try: filename = get_filename(file_path) logger.info('%s: readblock from server %d', file_path, settings.server_id) file_path = os.path.join(settings.data_dir, filename) # buffer = await read_file_block_from_fs(file_path) # stream = io.BytesIO(buffer) return FileResponse(file_path, media_type='application/octet-stream') except Exception as e: logger.exception(e) raise HTTPException(400, str(e))
async def delete_file(file_path: str): try: logger.info('delete: %s', file_path) start = time.time() result = await remove_file(file_path) deleted_count = sum(result) return { 'success': deleted_count == settings.primary + settings.parity, 'result': result, 'delay': time.time() - start } except Exception as e: logger.exception(e) raise HTTPException(400, str(e))
def __init__(self, send_message): """Create wave file writer""" super().__init__(send_message) try: WaveFileWriter.file_index = WaveFileWriter.file_index + 1 if WaveFileWriter.file_index > 99: WaveFileWriter.file_index = 1 self._file_name = ( f"{settings.recordings_path}{WaveFileWriter.file_index}-{int(time.time())}.wav" ) self._file = open(self._file_name, 'wb') logger.info("WaveFileWriter - Created file: %s", self._file_name) except OSError: logger.exception("WaveFileWriter - Failed to create file") self.on_error( "Engine: wave_file_writer - Message: Failed to create file")
async def write_file(file_path: str, file: UploadFile = File(...)): try: logger.info('write: %s', file_path) buffer = await file.read() file = generate_file(file_path, buffer) piece_map = [-1] * (settings.primary + settings.parity) result, delay = await process_file(file, piece_map) received_count = sum(result) return { 'success': received_count >= settings.primary, 'result': result, 'delay': delay } except Exception as e: logger.exception(e) raise HTTPException(400, str(e))
async def delete_file_block_in_fs(file_path, timestamp): async with write_file_block_in_fs_lock: filename = get_filename(file_path) new_file_path = os.path.join(settings.data_dir, filename) delete_flag = True try: async with aiofile.async_open(new_file_path, 'rb') as f: piece = pickle.loads(await f.read()) if piece.timestamp >= timestamp: delete_flag = False except Exception as e: logger.exception(e) pass logger.info('%s: deleteblock from server %d, flag=%d', file_path, settings.server_id, delete_flag) if delete_flag: os.remove(new_file_path) return delete_flag
async def write_file_block(file_path: str, request: Request): try: async with write_file_block_lock: new_buffer = await request.body() new_piece = pickle.loads(new_buffer) replace = True try: old_buffer = await read_file_block_from_fs(file_path) old_piece = pickle.loads(old_buffer) if new_piece.timestamp < old_piece.timestamp: replace = False except: pass if replace: await write_file_block_in_fs(file_path, new_buffer) else: logger.error('%s failed: writeblock into server %d, received file has smaller timestamp', file_path, settings.server_id) except Exception as e: logger.exception(e) raise HTTPException(400, str(e))
async def delete_file_block(server_id, file_path, timestamp): try: if server_id == settings.server_id: return await delete_file_block_in_fs(file_path, timestamp) else: port = settings.base_port + server_id url = 'http://%s:%d/deleteblock/%s?timestamp=%d' % (settings.host, port, file_path, timestamp) session = get_session() async with session.get(url) as resp: resp: aiohttp.ClientResponse if resp.status == 200: data = await resp.json() logger.info(data) if data['result'] is True: return True else: raise Exception(resp) except Exception as e: pass # logger.error('%s failed: receiveblock from server %d', file_path, server_id) logger.exception(e) return False
async def receive_file_block(server_id, file_path): try: if server_id == settings.server_id: buffer = await read_file_block_from_fs(file_path) piece = pickle.loads(buffer) logger.info('%s: receiveblock %d from server %d', file_path, piece.piece_id, server_id) return piece else: port = settings.base_port + server_id url = 'http://%s:%d/readblock/%s' % (settings.host, port, file_path) session = get_session() async with session.get(url) as resp: resp: aiohttp.ClientResponse if resp.status == 200: buffer = await resp.read() piece = pickle.loads(buffer) logger.info('%s: receiveblock %d from server %d', file_path, piece.piece_id, server_id) return piece else: raise Exception(resp) except Exception as e: logger.error('%s failed: receiveblock from server %d', file_path, server_id) logger.exception(e) return None