async def read_file(file_path: str, background_tasks: BackgroundTasks, stats: bool = False):
try:
tasks = []
start = time.time()
for i in range(settings.primary + settings.parity):
tasks.append(receive_file_block(i, file_path))
pieces = await asyncio.gather(*tasks)
transfer_time = time.time() - start
start = time.time()
file, piece_map = decode_data(pieces)
temp_file = tempfile.NamedTemporaryFile(delete=False)
async with aiofile.async_open(temp_file.name, 'wb') as f:
await f.write(file.buffer)
decode_time = time.time() - start
background_tasks.add_task(delete_temp_file, temp_file.name)
background_tasks.add_task(rebuild_redundancy, file, piece_map)
if stats:
return {
'delay': [transfer_time, decode_time]
}
else:
return FileResponse(temp_file.name, media_type='application/octet-stream')
except Exception as e:
logger.exception(e)
raise HTTPException(400, str(e))
async def send_file_block(server_id, file_path, piece):
try:
buffer = pickle.dumps(piece)
if server_id == settings.server_id:
await write_file_block_in_fs(file_path, buffer)
logger.info('%s: sendblock %d to server %d', file_path, piece.piece_id, server_id)
return True
else:
port = settings.base_port + server_id
url = 'http://%s:%d/writeblock/%s' % (settings.host, port, file_path)
headers = {
'content-type': 'application/octet-stream'
}
session = get_session()
async with session.post(url, data=buffer, headers=headers) as resp:
resp: aiohttp.ClientResponse
if resp.status == 200:
logger.info('%s: sendblock %d to server %d', file_path, piece.piece_id, server_id)
return True
else:
raise Exception(resp)
except Exception as e:
logger.error('%s failed: sendblock %d to server %d', file_path, piece.piece_id, server_id)
logger.exception(e)
return False
async def authenticate(self, socket_message: SocketJsonInputMessage):
"""Check if user is valid"""
client_id = socket_message.client_id
token = socket_message.access_token
processor_options = socket_message.data
# Try one token for all
if COMMON_TOKEN and token == COMMON_TOKEN:
self.is_authenticated = True
# Try user list
elif client_id and token:
if client_id in settings.user_tokens:
user_token = settings.user_tokens[client_id]
if user_token is not None and user_token == token:
self.is_authenticated = True
# Create processor
if self.is_authenticated:
try:
self.processor = ChunkProcessor(processor_name=None,
send_message=self.send_message, options=processor_options)
except RuntimeError:
logger.exception("ChunkProcessor - Failed to create processor")
await self.send_message(SocketErrorMessage(500,
"ChunkProcessorError", "Failed to create processor."))
else:
logger.warning("User %s failed to authenticate!", client_id)
await asyncio.sleep(3)
async def process(self, chunk: bytes):
"""Write chunks to file"""
try:
self._file.write(chunk)
except OSError:
logger.exception("WaveFileWriter - Failed to process")
self.on_error(
"Engine: wave_file_writer - Message: Failed to process")
async def delete_file_block(file_path: str, timestamp: int):
try:
async with write_file_block_lock:
result = await delete_file_block_in_fs(file_path, timestamp)
return {
'result': result
}
except Exception as e:
logger.exception(e)
raise HTTPException(400, str(e))
def _close_file(self):
"""Try to close file"""
try:
if self._file is not None and not self._file.closed:
self._file.close()
logger.info("WaveFileWriter - File closed: %s",
self._file_name)
except OSError:
logger.exception("WaveFileWriter - Failed to close file")
self.on_error(
"Engine: wave_file_writer - Message: Failed to close")
async def read_file_block(file_path: str):
try:
filename = get_filename(file_path)
logger.info('%s: readblock from server %d', file_path, settings.server_id)
file_path = os.path.join(settings.data_dir, filename)
# buffer = await read_file_block_from_fs(file_path)
# stream = io.BytesIO(buffer)
return FileResponse(file_path, media_type='application/octet-stream')
except Exception as e:
logger.exception(e)
raise HTTPException(400, str(e))
async def delete_file(file_path: str):
try:
logger.info('delete: %s', file_path)
start = time.time()
result = await remove_file(file_path)
deleted_count = sum(result)
return {
'success': deleted_count == settings.primary + settings.parity,
'result': result,
'delay': time.time() - start
}
except Exception as e:
logger.exception(e)
raise HTTPException(400, str(e))
def __init__(self, send_message):
"""Create wave file writer"""
super().__init__(send_message)
try:
WaveFileWriter.file_index = WaveFileWriter.file_index + 1
if WaveFileWriter.file_index > 99:
WaveFileWriter.file_index = 1
self._file_name = (
f"{settings.recordings_path}{WaveFileWriter.file_index}-{int(time.time())}.wav"
)
self._file = open(self._file_name, 'wb')
logger.info("WaveFileWriter - Created file: %s", self._file_name)
except OSError:
logger.exception("WaveFileWriter - Failed to create file")
self.on_error(
"Engine: wave_file_writer - Message: Failed to create file")
async def write_file(file_path: str, file: UploadFile = File(...)):
try:
logger.info('write: %s', file_path)
buffer = await file.read()
file = generate_file(file_path, buffer)
piece_map = [-1] * (settings.primary + settings.parity)
result, delay = await process_file(file, piece_map)
received_count = sum(result)
return {
'success': received_count >= settings.primary,
'result': result,
'delay': delay
}
except Exception as e:
logger.exception(e)
raise HTTPException(400, str(e))
async def delete_file_block_in_fs(file_path, timestamp):
async with write_file_block_in_fs_lock:
filename = get_filename(file_path)
new_file_path = os.path.join(settings.data_dir, filename)
delete_flag = True
try:
async with aiofile.async_open(new_file_path, 'rb') as f:
piece = pickle.loads(await f.read())
if piece.timestamp >= timestamp:
delete_flag = False
except Exception as e:
logger.exception(e)
pass
logger.info('%s: deleteblock from server %d, flag=%d', file_path, settings.server_id, delete_flag)
if delete_flag:
os.remove(new_file_path)
return delete_flag
async def write_file_block(file_path: str, request: Request):
try:
async with write_file_block_lock:
new_buffer = await request.body()
new_piece = pickle.loads(new_buffer)
replace = True
try:
old_buffer = await read_file_block_from_fs(file_path)
old_piece = pickle.loads(old_buffer)
if new_piece.timestamp < old_piece.timestamp:
replace = False
except:
pass
if replace:
await write_file_block_in_fs(file_path, new_buffer)
else:
logger.error('%s failed: writeblock into server %d, received file has smaller timestamp',
file_path, settings.server_id)
except Exception as e:
logger.exception(e)
raise HTTPException(400, str(e))
async def delete_file_block(server_id, file_path, timestamp):
try:
if server_id == settings.server_id:
return await delete_file_block_in_fs(file_path, timestamp)
else:
port = settings.base_port + server_id
url = 'http://%s:%d/deleteblock/%s?timestamp=%d' % (settings.host, port, file_path, timestamp)
session = get_session()
async with session.get(url) as resp:
resp: aiohttp.ClientResponse
if resp.status == 200:
data = await resp.json()
logger.info(data)
if data['result'] is True:
return True
else:
raise Exception(resp)
except Exception as e:
pass
# logger.error('%s failed: receiveblock from server %d', file_path, server_id)
logger.exception(e)
return False
async def receive_file_block(server_id, file_path):
try:
if server_id == settings.server_id:
buffer = await read_file_block_from_fs(file_path)
piece = pickle.loads(buffer)
logger.info('%s: receiveblock %d from server %d', file_path, piece.piece_id, server_id)
return piece
else:
port = settings.base_port + server_id
url = 'http://%s:%d/readblock/%s' % (settings.host, port, file_path)
session = get_session()
async with session.get(url) as resp:
resp: aiohttp.ClientResponse
if resp.status == 200:
buffer = await resp.read()
piece = pickle.loads(buffer)
logger.info('%s: receiveblock %d from server %d', file_path, piece.piece_id, server_id)
return piece
else:
raise Exception(resp)
except Exception as e:
logger.error('%s failed: receiveblock from server %d', file_path, server_id)
logger.exception(e)
return None
