def _synchronize_task_dir_p2(self, agent, local_td, generated_files, async_value_remote_td): """ Synchronizes the task directory with the remote agent, part 2 """ try: remote_td = copy.deepcopy(async_value_remote_td.value) except: print "An error occured while retrieving list of files in the task dir from remote agent" return if remote_td is None: # sync disabled for this Agent return to_update, to_delete = directory_compare_from_hash(local_td, remote_td) tmpfile = tempfile.TemporaryFile() tar = tarfile.open(fileobj=tmpfile, mode='w:gz') for path in to_update: # be a little safe about what the agent returns... if os.path.relpath(os.path.join(get_tasks_directory(), path), get_tasks_directory()) == path and ".." not in path: if path in generated_files: # the file do not really exists on disk, it was generated info = tarfile.TarInfo(name=path) info.size = generated_files[path].len info.mode = 0o777 tar.addfile(tarinfo=info, fileobj=generated_files[path]) else: # the file really exists on disk tar.add(arcname=path, name=os.path.join(get_tasks_directory(), path)) else: print "Agent returned non-safe file path: " + path tar.close() tmpfile.flush() tmpfile.seek(0) # sync the agent async_update = rpyc.async(agent.root.update_task_directory) # do not forget to close the file async_update(tmpfile, to_delete).add_callback(lambda r: tmpfile.close())
def test_directory_compare_from_hash(self): test_dir = os.path.join(self.dir_path, "test2") # Create data os.mkdir(test_dir) os.mkdir(os.path.join(test_dir, "subdir")) open(os.path.join(test_dir, "file1"), "w").write("random text 1") open(os.path.join(test_dir, "file2"), "w").write("random text 2") open(os.path.join(test_dir, "subdir", "file3"), "w").write("random text 3") open(os.path.join(test_dir, "file4"), "w").write("random text 4") open(os.path.join(test_dir, "file5"), "w").write("random text 5") open(os.path.join(test_dir, "file6"), "w").write("random text 6") l1 = directory_content_with_hash(test_dir) l2 = copy.deepcopy(l1) # Pertub the data l2["file1"] = (l2["file1"][0], 0) l2["file2"] = ("not a valid hash", l2["file2"]) l2["file4"] = ("not a valid hash", 0) del l2["file5"] # Compare and test to_update, to_delete = directory_compare_from_hash(l2, l1) assert set(to_update) == set(["file1", "file2", "file4"]) assert set(to_delete) == set(["file5"])
def _try_synchronize_task_dir(self): """ Check if the remote tasks dirs (on the remote agents) should be updated """ if self._closed: return current_content_in_task_directory = directory_content_with_hash(get_tasks_directory()) changed, deleted = directory_compare_from_hash(current_content_in_task_directory, self._last_content_in_task_directory) if len(changed) != 0 or len(deleted) != 0: self._last_content_in_task_directory = current_content_in_task_directory for agent in self._agents: if agent is not None: self._synchronize_task_dir(agent) if not self._is_testing: threading.Timer(30, self._try_synchronize_task_dir).start()