def test_directory_compare_from_hash(self):
test_dir = os.path.join(self.dir_path, "test2")
# Create data
os.mkdir(test_dir)
os.mkdir(os.path.join(test_dir, "subdir"))
with open(os.path.join(test_dir, "file1"), "w") as f:
f.write("random text 1")
with open(os.path.join(test_dir, "file2"), "w") as f:
f.write("random text 2")
with open(os.path.join(test_dir, "subdir", "file3"), "w") as f:
f.write("random text 3")
with open(os.path.join(test_dir, "file4"), "w") as f:
f.write("random text 4")
with open(os.path.join(test_dir, "file5"), "w") as f:
f.write("random text 5")
with open(os.path.join(test_dir, "file6"), "w") as f:
f.write("random text 6")
l1 = directory_content_with_hash(test_dir)
l2 = copy.deepcopy(l1)
# Perturb the data
l2["file1"] = (l2["file1"][0], 0)
l2["file2"] = ("not a valid hash", l2["file2"])
l2["file4"] = ("not a valid hash", 0)
del l2["file5"]
# Compare and test
to_update, to_delete = directory_compare_from_hash(l2, l1)
assert set(to_update) == set(["file1", "file2", "file4"])
assert set(to_delete) == set(["file5"])
def test_directory_compare_from_hash(self):
test_dir = os.path.join(self.dir_path, "test2")
# Create data
os.mkdir(test_dir)
os.mkdir(os.path.join(test_dir, "subdir"))
open(os.path.join(test_dir, "file1"), "w").write("random text 1")
open(os.path.join(test_dir, "file2"), "w").write("random text 2")
open(os.path.join(test_dir, "subdir", "file3"), "w").write("random text 3")
open(os.path.join(test_dir, "file4"), "w").write("random text 4")
open(os.path.join(test_dir, "file5"), "w").write("random text 5")
open(os.path.join(test_dir, "file6"), "w").write("random text 6")
l1 = directory_content_with_hash(test_dir)
l2 = copy.deepcopy(l1)
# Pertub the data
l2["file1"] = (l2["file1"][0], 0)
l2["file2"] = ("not a valid hash", l2["file2"])
l2["file4"] = ("not a valid hash", 0)
del l2["file5"]
# Compare and test
to_update, to_delete = directory_compare_from_hash(l2, l1)
assert set(to_update) == set(["file1", "file2", "file4"])
assert set(to_delete) == set(["file5"])
def _synchronize_task_dir_p2(self, agent, local_td, generated_files, async_value_remote_td):
""" Synchronizes the task directory with the remote agent, part 2 """
try:
remote_td = copy.deepcopy(async_value_remote_td.value)
except:
print "An error occured while retrieving list of files in the task dir from remote agent"
return
if remote_td is None: # sync disabled for this Agent
return
to_update, to_delete = directory_compare_from_hash(local_td, remote_td)
tmpfile = tempfile.TemporaryFile()
tar = tarfile.open(fileobj=tmpfile, mode='w:gz')
for path in to_update:
# be a little safe about what the agent returns...
if os.path.relpath(os.path.join(self._task_directory, path), self._task_directory) == path and ".." not in path:
if path in generated_files: # the file do not really exists on disk, it was generated
info = tarfile.TarInfo(name=path)
info.size = generated_files[path].len
info.mode = 0o777
tar.addfile(tarinfo=info, fileobj=generated_files[path])
else: # the file really exists on disk
tar.add(arcname=path, name=os.path.join(self._task_directory, path))
else:
print "Agent returned non-safe file path: " + path
tar.close()
tmpfile.flush()
tmpfile.seek(0)
# sync the agent
async_update = rpyc.async(agent.root.update_task_directory)
# do not forget to close the file
async_update(tmpfile, to_delete).add_callback(lambda r: self._synchronize_task_dir_done(agent, tmpfile))
def _try_synchronize_task_dir(self):
""" Check if the remote tasks dirs (on the remote agents) should be updated """
if self._closed:
return
current_content_in_task_directory = directory_content_with_hash(self._task_directory)
changed, deleted = directory_compare_from_hash(current_content_in_task_directory, self._last_content_in_task_directory)
if len(changed) != 0 or len(deleted) != 0:
self._last_content_in_task_directory = current_content_in_task_directory
for agent in self._agents:
if agent is not None:
self._synchronize_task_dir(agent)
if not self._is_testing:
self._timers[self._try_synchronize_task_dir] = threading.Timer(30, self._try_synchronize_task_dir)
self._timers[self._try_synchronize_task_dir].start()
def _synchronize_task_dir_p2(self, agent, local_td, generated_files,
async_value_remote_td):
""" Synchronizes the task directory with the remote agent, part 2 """
try:
remote_td = copy.deepcopy(async_value_remote_td.value)
except:
self._logger.exception(
"An error occured while retrieving list of files in the task dir from remote agent",
exc_info=True)
return
if remote_td is None: # sync disabled for this Agent
return
to_update, to_delete = directory_compare_from_hash(local_td, remote_td)
tmpfile = tempfile.TemporaryFile()
tar = tarfile.open(fileobj=tmpfile, mode='w:gz')
for path in to_update:
# be a little safe about what the agent returns...
if os.path.relpath(
os.path.join(self._task_directory, path),
self._task_directory) == path and ".." not in path:
if path in generated_files: # the file do not really exists on disk, it was generated
info = tarfile.TarInfo(name=path)
info.size = generated_files[path].len
info.mode = 0o777
tar.addfile(tarinfo=info, fileobj=generated_files[path])
else: # the file really exists on disk
tar.add(arcname=path,
name=os.path.join(self._task_directory, path))
else:
self._logger.critical("Agent returned non-safe file path: " +
path)
tar.close()
tmpfile.flush()
tmpfile.seek(0)
# sync the agent
async_update = rpyc. async (agent.root.update_task_directory)
# do not forget to close the file
async_update(tmpfile, to_delete).add_callback(
lambda r: self._synchronize_task_dir_done(agent, tmpfile))
def _try_synchronize_task_dir(self):
""" Check if the remote tasks dirs (on the remote agents) should be updated """
if self._closed:
return
current_content_in_task_directory = directory_content_with_hash(
self._task_directory)
changed, deleted = directory_compare_from_hash(
current_content_in_task_directory,
self._last_content_in_task_directory)
if len(changed) != 0 or len(deleted) != 0:
self._last_content_in_task_directory = current_content_in_task_directory
for agent in self._agents:
if agent is not None:
self._synchronize_task_dir(agent)
if not self._is_testing:
self._timers[self._try_synchronize_task_dir] = threading.Timer(
30, self._try_synchronize_task_dir)
self._timers[self._try_synchronize_task_dir].start()