def _synchronize_task_dir_p2(self, agent, local_td, generated_files, async_value_remote_td):
""" Synchronizes the task directory with the remote agent, part 2 """
try:
remote_td = copy.deepcopy(async_value_remote_td.value)
except:
print "An error occured while retrieving list of files in the task dir from remote agent"
return
if remote_td is None: # sync disabled for this Agent
return
to_update, to_delete = directory_compare_from_hash(local_td, remote_td)
tmpfile = tempfile.TemporaryFile()
tar = tarfile.open(fileobj=tmpfile, mode='w:gz')
for path in to_update:
# be a little safe about what the agent returns...
if os.path.relpath(os.path.join(get_tasks_directory(), path), get_tasks_directory()) == path and ".." not in path:
if path in generated_files: # the file do not really exists on disk, it was generated
info = tarfile.TarInfo(name=path)
info.size = generated_files[path].len
info.mode = 0o777
tar.addfile(tarinfo=info, fileobj=generated_files[path])
else: # the file really exists on disk
tar.add(arcname=path, name=os.path.join(get_tasks_directory(), path))
else:
print "Agent returned non-safe file path: " + path
tar.close()
tmpfile.flush()
tmpfile.seek(0)
# sync the agent
async_update = rpyc.async(agent.root.update_task_directory)
# do not forget to close the file
async_update(tmpfile, to_delete).add_callback(lambda r: tmpfile.close())
def test_directory_compare_from_hash(self):
test_dir = os.path.join(self.dir_path, "test2")
# Create data
os.mkdir(test_dir)
os.mkdir(os.path.join(test_dir, "subdir"))
open(os.path.join(test_dir, "file1"), "w").write("random text 1")
open(os.path.join(test_dir, "file2"), "w").write("random text 2")
open(os.path.join(test_dir, "subdir", "file3"), "w").write("random text 3")
open(os.path.join(test_dir, "file4"), "w").write("random text 4")
open(os.path.join(test_dir, "file5"), "w").write("random text 5")
open(os.path.join(test_dir, "file6"), "w").write("random text 6")
l1 = directory_content_with_hash(test_dir)
l2 = copy.deepcopy(l1)
# Pertub the data
l2["file1"] = (l2["file1"][0], 0)
l2["file2"] = ("not a valid hash", l2["file2"])
l2["file4"] = ("not a valid hash", 0)
del l2["file5"]
# Compare and test
to_update, to_delete = directory_compare_from_hash(l2, l1)
assert set(to_update) == set(["file1", "file2", "file4"])
assert set(to_delete) == set(["file5"])
def _try_synchronize_task_dir(self):
""" Check if the remote tasks dirs (on the remote agents) should be updated """
if self._closed:
return
current_content_in_task_directory = directory_content_with_hash(get_tasks_directory())
changed, deleted = directory_compare_from_hash(current_content_in_task_directory, self._last_content_in_task_directory)
if len(changed) != 0 or len(deleted) != 0:
self._last_content_in_task_directory = current_content_in_task_directory
for agent in self._agents:
if agent is not None:
self._synchronize_task_dir(agent)
if not self._is_testing:
threading.Timer(30, self._try_synchronize_task_dir).start()