def test_simple_backup(self):
node_handler_process = NodeHandlerProcess('localhost',
TestSidecar.PORT,
'/tmp/example',
'/tmp/backup_output/out',
'dummy_checksum')
sleep(5)
node_handler_process()
expected_file = BackupFile.create_from_path('/tmp/example',
"/tmp/backup_output/out2")
backup_file = BackupFile("/tmp/backup_output/out")
self.assertEqual(expected_file.get_hash(), backup_file.get_hash())
def setUp(self) -> None:
try:
from pytest_cov.embed import cleanup_on_sigterm
except ImportError:
pass
else:
cleanup_on_sigterm()
shutil.rmtree('/tmp/backup_output', ignore_errors=True)
os.mkdir('/tmp/backup_output')
with open('/tmp/example', 'w') as example_file:
example_file.write("asd")
BackupFile.create_from_path('/tmp/example', "/tmp/backup_output/out2")
SidecarProcess.logger = logging.getLogger("dummy")
self.sidecar_process = SidecarProcess(TestSidecar.PORT, 3)
self.p = Process(target=self.sidecar_process)
self.p.start()
def __call__(self, *args, **kwargs):
MockNodeHandler.BARRIER.wait()
bf = BackupFile.create_from_path(MockNodeHandler.PATH_TO_BACKUP,
self.write_file_path)
if bf.get_hash() == self.previous_checksum:
open(self.write_file_path + ".SAME", "w").close()
else:
open(self.write_file_path + ".CORRECT", "w").close()
def test_backup_same_checksum(self):
expected_file = BackupFile.create_from_path('/tmp/example',
"/tmp/backup_output/out2")
node_handler_process = NodeHandlerProcess('localhost',
TestSidecar.PORT,
'/tmp/example',
'/tmp/backup_output/out',
expected_file.get_hash())
sleep(5)
node_handler_process()
self.assertTrue(os.path.exists('/tmp/backup_output/out.SAME'))
def __handle_client_connection(client_sock, backup_no: int):
"""
Read message from a specific client socket and closes the socket
If a problem arises in the communication with the client, the
client socket will also be closed
"""
socket_transferer = BlockingSocketTransferer(client_sock)
try:
msg = socket_transferer.receive_plain_text()
msg = json.loads(msg)
path, previous_checksum = msg['path'], msg['checksum']
SidecarProcess.logger.debug("Previous checksum for path %s is '%s'" % (path, previous_checksum))
except (OSError, TimeoutError) as e:
SidecarProcess.logger.exception("Error while reading socket %s: %s" % (client_sock, e))
socket_transferer.abort()
return
try:
backup_file = BackupFile.create_from_path(path, TMP_BACKUP_PATH % backup_no)
except Exception:
SidecarProcess.logger.exception("Error while making backup file")
socket_transferer.abort()
return
file_checksum = backup_file.get_hash()
if file_checksum == previous_checksum:
SidecarProcess.logger.info("Previous checksum equals to actual data, skipping backup")
socket_transferer.send_plain_text("SAME")
socket_transferer.abort()
return
else:
socket_transferer.send_plain_text("DIFF")
try:
socket_transferer.send_file(TMP_BACKUP_PATH % backup_no)
SidecarProcess.logger.debug("Backup file sent")
socket_transferer.send_plain_text(file_checksum)
except Exception as e:
SidecarProcess.logger.exception("Error while writing socket %s: %s" % (client_sock, e))
socket_transferer.abort()
return
finally:
socket_transferer.close()
return
def _dispatch_running_tasks(self):
"""
Handles running tasks
"""
now_running_tasks = {}
for node_data, task in self.running_tasks.items():
if not task.is_running():
if task.backup_is_correct():
ft = FinishedTask(
result_path=task.write_file_path,
kb_size=os.path.getsize(task.write_file_path) / 1024,
timestamp=datetime.now(),
checksum=BackupFile(task.write_file_path).get_hash())
self.database.register_finished_task(
node_data[0], node_data[1], ft)
BackupScheduler.logger.info(
"Backup for node %s and path %s finished succesfully" %
node_data)
self._reload_schedule()
self._clean_backup_path()
elif task.backup_is_same():
ft = self.database.get_node_finished_tasks(
node_data[0], node_data[1])[0]
ft = FinishedTask(result_path=ft.result_path,
kb_size=ft.kb_size,
timestamp=datetime.now(),
checksum=ft.checksum)
self.database.register_finished_task(
node_data[0], node_data[1], ft)
BackupScheduler.logger.info(
"Backup for node %s and path %s finished succesfully" %
node_data)
self._reload_schedule()
self._clean_backup_path()
else:
BackupScheduler.logger.error(
"Backup for node %s and path %s failed" % node_data)
else:
now_running_tasks[node_data] = task
self.running_tasks = now_running_tasks
def __call__(self) -> NoReturn:
"""
Code for running the handler in a new process
The process works this way:
1. Connects to node sidecar asking for node_path compressed
2. If the backup is the same as previous checksum, writes a .SAME file
3. Downloads the file saving it in write file path
3.1. At start it writes an empty file named self.write_file_path but ending with .WIP
3.2. Starts saving the backup in a file located in self.write_file_path
3.3. When the backup is saved saves an empty file named self.write_file_path but ending with .CORRECT
3.4. Deletes the .WIP file
4. Ｓｅｐｐｕｋｕ
"""
NodeHandlerProcess.logger.debug(
"Starting node handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((self.node_address, self.node_port))
socket_transferer = BlockingSocketTransferer(sock)
socket_transferer.send_plain_text(
json.dumps({
"checksum": self.previous_checksum,
"path": self.node_path
}))
except Exception as e:
NodeHandlerProcess.logger.exception(
"Error while writing socket %s: %s" % (sock, e))
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
return
msg = socket_transferer.receive_plain_text()
if msg == "SAME":
NodeHandlerProcess.logger.debug("The backup was the same")
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
open(SAME_FILE_FORMAT % self.write_file_path, 'w').close()
return
if msg == "ABORT":
NodeHandlerProcess.logger.error("Abort order sent from sidecar")
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
return
open(WIP_FILE_FORMAT % self.write_file_path, 'w').close()
data_file = open(self.write_file_path, 'ab')
try:
socket_transferer.receive_file_data(data_file)
NodeHandlerProcess.logger.debug("File data received")
checksum = socket_transferer.receive_plain_text()
except Exception as e:
NodeHandlerProcess.logger.exception(
"Error while reading socket %s: %s" % (sock, e))
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
return
data_file.close()
backup_file = BackupFile(self.write_file_path)
if backup_file.get_hash() == checksum:
NodeHandlerProcess.logger.debug("Backup checksum: %s" % checksum)
else:
NodeHandlerProcess.logger.error(
"Error verifying checksum. Local: %s vs Server: %s" %
(backup_file.get_hash(), checksum))
return
open(CORRECT_FILE_FORMAT % self.write_file_path, 'w').close()
os.remove(WIP_FILE_FORMAT % self.write_file_path)
NodeHandlerProcess.logger.info(
"Terminating handler for node %s:%d and path %s" %
(self.node_address, self.node_port, self.node_path))
def test_generate_different_same_hash(self):
backup_file = BackupFile.create_from_path('/tmp/test_path',
'/tmp/file.tgz')
backup_file2 = BackupFile.create_from_path('/tmp/test_path',
'/tmp/file2.tgz')
self.assertEqual(backup_file2.get_hash(), backup_file.get_hash())
def test_load_and_create_same_hash(self):
backup_file = BackupFile.create_from_path('/tmp/test_path',
'/tmp/file.tgz')
backup_file2 = BackupFile('/tmp/file.tgz')
self.assertEqual(backup_file2.get_hash(), backup_file.get_hash())