def test_send_text(self):
self.p = Process(target=message_sender, args=(self.barrier, TestBlockingSocketTransferer.TEST_PORT))
self.p.start()
self.barrier.wait()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', TestBlockingSocketTransferer.TEST_PORT))
socket_transferer = BlockingSocketTransferer(sock)
self.assertEqual(socket_transferer.receive_plain_text(), "Hola uacho")
socket_transferer.close()
def test_detect_backup_scheduler_dead(self):
self.barrier.wait()
self.backup_scheduler_send.close()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', TestClientListener.PORT))
socket_transferer = BlockingSocketTransferer(sock)
socket_transferer.send_plain_text(
'{"command": "dummy", "args": {"one": "one"}}')
msg = socket_transferer.receive_plain_text()
self.assertEqual("ABORT", msg)
sleep(1)
self.assertTrue(not self.p.is_alive())
def test_detect_socket_dead(self):
self.barrier.wait()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', TestClientListener.PORT))
socket_transferer = BlockingSocketTransferer(sock)
socket_transferer.send_plain_text(
'{"command": "dummy", "args": {"one": "one"}}')
command, args = self.backup_scheduler_recv.recv()
self.assertEqual(command, 'dummy')
self.assertEqual(args, {"one": "one"})
sock.shutdown(socket.SHUT_RDWR)
sock.close()
self.backup_scheduler_send.send(("OK", {}))
sleep(1)
self.assertTrue(not self.p.is_alive())
def test_send_file(self):
with open('/tmp/big_dummy_file_test', 'wb') as dummy_file:
for i in range(100000):
dummy_file.write(("%d%d%d" % (i, i, i)).encode('utf-8'))
sha256 = hashlib.sha256()
with open('/tmp/big_dummy_file_test', 'rb') as dummy_file:
while True:
data = dummy_file.read(2048)
if not data:
break
sha256.update(data)
original_hash = sha256.hexdigest()
self.p = Process(target=file_sender, args=(self.barrier, TestBlockingSocketTransferer.TEST_PORT,
'/tmp/big_dummy_file_test'))
self.p.start()
self.barrier.wait()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', TestBlockingSocketTransferer.TEST_PORT))
socket_transferer = BlockingSocketTransferer(sock)
with open('/tmp/big_dummy_file_test_out', 'wb') as write_file:
socket_transferer.receive_file_data(write_file)
sha256 = hashlib.sha256()
with open('/tmp/big_dummy_file_test_out', 'rb') as dummy_file:
while True:
data = dummy_file.read(2048)
if not data:
break
sha256.update(data)
self.assertEqual(sha256.hexdigest(), original_hash)
os.remove('/tmp/big_dummy_file_test')
os.remove('/tmp/big_dummy_file_test_out')
socket_transferer.close()
def file_sender(barrier, port, input_file):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', port))
sock.listen(1)
barrier.wait()
c, addr = sock.accept()
transferer = BlockingSocketTransferer(c)
transferer.send_file(input_file)
transferer.close()
def test_fail_to_receive_file(self):
sleep(5)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', TestSidecar.PORT))
socket_transferer = BlockingSocketTransferer(sock)
socket_transferer.send_plain_text(
json.dumps({
"checksum": "",
"path": '/tmp/example'
}))
_ = socket_transferer.receive_plain_text()
socket_transferer.close()
sleep(5)
def test_send_and_receive_command(self):
self.barrier.wait()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('localhost', TestClientListener.PORT))
socket_transferer = BlockingSocketTransferer(sock)
socket_transferer.send_plain_text(
'{"command": "dummy", "args": {"one": "one"}}')
command, args = self.backup_scheduler_recv.recv()
self.assertEqual(command, 'dummy')
self.assertEqual(args, {"one": "one"})
self.backup_scheduler_send.send(("OK", {}))
msg = socket_transferer.receive_plain_text()
self.assertEqual(json.loads(msg), {"message": "OK", "data": {}})
socket_transferer.close()
def __call__(self) -> NoReturn:
"""
Code for running the handler in a new process
The process works this way:
1. Connects to node sidecar asking for node_path compressed
2. If the backup is the same as previous checksum, writes a .SAME file
3. Downloads the file saving it in write file path
3.1. At start it writes an empty file named self.write_file_path but ending with .WIP
3.2. Starts saving the backup in a file located in self.write_file_path
3.3. When the backup is saved saves an empty file named self.write_file_path but ending with .CORRECT
3.4. Deletes the .WIP file
4. Ｓｅｐｐｕｋｕ
"""
NodeHandlerProcess.logger.debug(
"Starting node handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((self.node_address, self.node_port))
socket_transferer = BlockingSocketTransferer(sock)
socket_transferer.send_plain_text(
json.dumps({
"checksum": self.previous_checksum,
"path": self.node_path
}))
except Exception as e:
NodeHandlerProcess.logger.exception(
"Error while writing socket %s: %s" % (sock, e))
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
return
msg = socket_transferer.receive_plain_text()
if msg == "SAME":
NodeHandlerProcess.logger.debug("The backup was the same")
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
open(SAME_FILE_FORMAT % self.write_file_path, 'w').close()
return
if msg == "ABORT":
NodeHandlerProcess.logger.error("Abort order sent from sidecar")
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
return
open(WIP_FILE_FORMAT % self.write_file_path, 'w').close()
data_file = open(self.write_file_path, 'ab')
try:
socket_transferer.receive_file_data(data_file)
NodeHandlerProcess.logger.debug("File data received")
checksum = socket_transferer.receive_plain_text()
except Exception as e:
NodeHandlerProcess.logger.exception(
"Error while reading socket %s: %s" % (sock, e))
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
return
data_file.close()
backup_file = BackupFile(self.write_file_path)
if backup_file.get_hash() == checksum:
NodeHandlerProcess.logger.debug("Backup checksum: %s" % checksum)
else:
NodeHandlerProcess.logger.error(
"Error verifying checksum. Local: %s vs Server: %s" %
(backup_file.get_hash(), checksum))
return
open(CORRECT_FILE_FORMAT % self.write_file_path, 'w').close()
os.remove(WIP_FILE_FORMAT % self.write_file_path)
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
def __handle_client_connection(self, client_sock):
"""
Read message from a specific client socket and closes the socket
If a problem arises in the communication with the client, the
client socket will also be closed
"""
socket_transferer = BlockingSocketTransferer(client_sock)
try:
msg = socket_transferer.receive_plain_text()
except Exception:
ClientListener.logger.exception("Error while reading socket {}".format(client_sock))
socket_transferer.abort()
return
try:
parsed_msg = json.loads(msg)
except json.decoder.JSONDecodeError as e:
ClientListener.logger.exception("Error parsing json")
socket_transferer.send_plain_text(json.dumps({"message": "ERROR", "data": str(e)}))
socket_transferer.close()
return
self.backup_scheduler_write.send((parsed_msg['command'], parsed_msg['args']))
try:
message, data = self.backup_scheduler_read.recv()
except EOFError as e:
ClientListener.logger.exception("Backup scheduler death")
socket_transferer.abort()
raise e
try:
socket_transferer.send_plain_text(json.dumps({"message": message, "data": data}))
except Exception:
ClientListener.logger.exception("Error writing through socket")
socket_transferer.abort()
return
socket_transferer.close()
def __handle_client_connection(client_sock, backup_no: int):
"""
Read message from a specific client socket and closes the socket
If a problem arises in the communication with the client, the
client socket will also be closed
"""
socket_transferer = BlockingSocketTransferer(client_sock)
try:
msg = socket_transferer.receive_plain_text()
msg = json.loads(msg)
path, previous_checksum = msg['path'], msg['checksum']
SidecarProcess.logger.debug("Previous checksum for path %s is '%s'" % (path, previous_checksum))
except (OSError, TimeoutError) as e:
SidecarProcess.logger.exception("Error while reading socket %s: %s" % (client_sock, e))
socket_transferer.abort()
return
try:
backup_file = BackupFile.create_from_path(path, TMP_BACKUP_PATH % backup_no)
except Exception:
SidecarProcess.logger.exception("Error while making backup file")
socket_transferer.abort()
return
file_checksum = backup_file.get_hash()
if file_checksum == previous_checksum:
SidecarProcess.logger.info("Previous checksum equals to actual data, skipping backup")
socket_transferer.send_plain_text("SAME")
socket_transferer.abort()
return
else:
socket_transferer.send_plain_text("DIFF")
try:
socket_transferer.send_file(TMP_BACKUP_PATH % backup_no)
SidecarProcess.logger.debug("Backup file sent")
socket_transferer.send_plain_text(file_checksum)
except Exception as e:
SidecarProcess.logger.exception("Error while writing socket %s: %s" % (client_sock, e))
socket_transferer.abort()
return
finally:
socket_transferer.close()
return
parser = argparse.ArgumentParser(
description='Sends a command to backup server')
parser.add_argument('--port',
required=True,
type=int,
help='the port of the the server')
parser.add_argument('--address',
required=True,
help='the address of the server')
parser.add_argument('--command', required=True, help='the command to run')
parser.add_argument('--args', required=True, help='the args dict')
args = parser.parse_args()
port = args.port
address = args.address
command = args.command
dict_arguments = json.loads(args.args)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((address, port))
socket_transferer = BlockingSocketTransferer(sock)
socket_transferer.send_plain_text(
json.dumps({
"command": command,
"args": dict_arguments
}))
msg = socket_transferer.receive_plain_text()
print(msg)
socket_transferer.close()