def test_directory_compare_from_hash(self):
        test_dir = os.path.join(self.dir_path, "test2")

        # Create data
        os.mkdir(test_dir)
        os.mkdir(os.path.join(test_dir, "subdir"))

        with open(os.path.join(test_dir, "file1"), "w") as f:
            f.write("random text 1")
        with open(os.path.join(test_dir, "file2"), "w") as f:
            f.write("random text 2")
        with open(os.path.join(test_dir, "subdir", "file3"), "w") as f:
            f.write("random text 3")
        with open(os.path.join(test_dir, "file4"), "w") as f:
            f.write("random text 4")
        with open(os.path.join(test_dir, "file5"), "w") as f:
            f.write("random text 5")
        with open(os.path.join(test_dir, "file6"), "w") as f:
            f.write("random text 6")

        l1 = directory_content_with_hash(test_dir)
        l2 = copy.deepcopy(l1)

        # Perturb the data
        l2["file1"] = (l2["file1"][0], 0)
        l2["file2"] = ("not a valid hash", l2["file2"])
        l2["file4"] = ("not a valid hash", 0)
        del l2["file5"]

        # Compare and test
        to_update, to_delete = directory_compare_from_hash(l2, l1)
        assert set(to_update) == set(["file1", "file2", "file4"])
        assert set(to_delete) == set(["file5"])
Ejemplo n.º 2
0
    def test_directory_compare_from_hash(self):
        test_dir = os.path.join(self.dir_path, "test2")

        # Create data
        os.mkdir(test_dir)
        os.mkdir(os.path.join(test_dir, "subdir"))
        open(os.path.join(test_dir, "file1"), "w").write("random text 1")
        open(os.path.join(test_dir, "file2"), "w").write("random text 2")
        open(os.path.join(test_dir, "subdir", "file3"), "w").write("random text 3")
        open(os.path.join(test_dir, "file4"), "w").write("random text 4")
        open(os.path.join(test_dir, "file5"), "w").write("random text 5")
        open(os.path.join(test_dir, "file6"), "w").write("random text 6")

        l1 = directory_content_with_hash(test_dir)
        l2 = copy.deepcopy(l1)

        # Pertub the data
        l2["file1"] = (l2["file1"][0], 0)
        l2["file2"] = ("not a valid hash", l2["file2"])
        l2["file4"] = ("not a valid hash", 0)
        del l2["file5"]

        # Compare and test
        to_update, to_delete = directory_compare_from_hash(l2, l1)
        assert set(to_update) == set(["file1", "file2", "file4"])
        assert set(to_delete) == set(["file5"])
    def _synchronize_task_dir_p2(self, agent, local_td, generated_files, async_value_remote_td):
        """ Synchronizes the task directory with the remote agent, part 2 """
        try:
            remote_td = copy.deepcopy(async_value_remote_td.value)
        except:
            print "An error occured while retrieving list of files in the task dir from remote agent"
            return

        if remote_td is None:  # sync disabled for this Agent
            return

        to_update, to_delete = directory_compare_from_hash(local_td, remote_td)
        tmpfile = tempfile.TemporaryFile()
        tar = tarfile.open(fileobj=tmpfile, mode='w:gz')
        for path in to_update:
            # be a little safe about what the agent returns...
            if os.path.relpath(os.path.join(self._task_directory, path), self._task_directory) == path and ".." not in path:
                if path in generated_files:  # the file do not really exists on disk, it was generated
                    info = tarfile.TarInfo(name=path)
                    info.size = generated_files[path].len
                    info.mode = 0o777
                    tar.addfile(tarinfo=info, fileobj=generated_files[path])
                else:  # the file really exists on disk
                    tar.add(arcname=path, name=os.path.join(self._task_directory, path))
            else:
                print "Agent returned non-safe file path: " + path
        tar.close()
        tmpfile.flush()
        tmpfile.seek(0)

        # sync the agent
        async_update = rpyc.async(agent.root.update_task_directory)
        # do not forget to close the file
        async_update(tmpfile, to_delete).add_callback(lambda r: self._synchronize_task_dir_done(agent, tmpfile))
    def _try_synchronize_task_dir(self):
        """ Check if the remote tasks dirs (on the remote agents) should be updated """
        if self._closed:
            return

        current_content_in_task_directory = directory_content_with_hash(self._task_directory)
        changed, deleted = directory_compare_from_hash(current_content_in_task_directory, self._last_content_in_task_directory)
        if len(changed) != 0 or len(deleted) != 0:
            self._last_content_in_task_directory = current_content_in_task_directory
            for agent in self._agents:
                if agent is not None:
                    self._synchronize_task_dir(agent)

        if not self._is_testing:
            self._timers[self._try_synchronize_task_dir] = threading.Timer(30, self._try_synchronize_task_dir)
            self._timers[self._try_synchronize_task_dir].start()
Ejemplo n.º 5
0
    def _synchronize_task_dir_p2(self, agent, local_td, generated_files,
                                 async_value_remote_td):
        """ Synchronizes the task directory with the remote agent, part 2 """
        try:
            remote_td = copy.deepcopy(async_value_remote_td.value)
        except:
            self._logger.exception(
                "An error occured while retrieving list of files in the task dir from remote agent",
                exc_info=True)
            return

        if remote_td is None:  # sync disabled for this Agent
            return

        to_update, to_delete = directory_compare_from_hash(local_td, remote_td)
        tmpfile = tempfile.TemporaryFile()
        tar = tarfile.open(fileobj=tmpfile, mode='w:gz')
        for path in to_update:
            # be a little safe about what the agent returns...
            if os.path.relpath(
                    os.path.join(self._task_directory, path),
                    self._task_directory) == path and ".." not in path:
                if path in generated_files:  # the file do not really exists on disk, it was generated
                    info = tarfile.TarInfo(name=path)
                    info.size = generated_files[path].len
                    info.mode = 0o777
                    tar.addfile(tarinfo=info, fileobj=generated_files[path])
                else:  # the file really exists on disk
                    tar.add(arcname=path,
                            name=os.path.join(self._task_directory, path))
            else:
                self._logger.critical("Agent returned non-safe file path: " +
                                      path)
        tar.close()
        tmpfile.flush()
        tmpfile.seek(0)

        # sync the agent
        async_update = rpyc. async (agent.root.update_task_directory)
        # do not forget to close the file
        async_update(tmpfile, to_delete).add_callback(
            lambda r: self._synchronize_task_dir_done(agent, tmpfile))
Ejemplo n.º 6
0
    def _try_synchronize_task_dir(self):
        """ Check if the remote tasks dirs (on the remote agents) should be updated """
        if self._closed:
            return

        current_content_in_task_directory = directory_content_with_hash(
            self._task_directory)
        changed, deleted = directory_compare_from_hash(
            current_content_in_task_directory,
            self._last_content_in_task_directory)
        if len(changed) != 0 or len(deleted) != 0:
            self._last_content_in_task_directory = current_content_in_task_directory
            for agent in self._agents:
                if agent is not None:
                    self._synchronize_task_dir(agent)

        if not self._is_testing:
            self._timers[self._try_synchronize_task_dir] = threading.Timer(
                30, self._try_synchronize_task_dir)
            self._timers[self._try_synchronize_task_dir].start()