def test_sync_nodes_only_diff(ray_start_2_cpus, temp_data_dirs): """Check that only differing files are synced between nodes""" tmp_source, tmp_target = temp_data_dirs # Sanity check assert_file(True, tmp_source, "level0.txt") assert_file(True, tmp_source, "subdir/level1.txt") assert_file(False, tmp_target, "level0.txt") assert_file(False, tmp_target, "level0_new.txt") node_ip = ray.util.get_node_ip_address() _sync_dir_between_different_nodes( source_ip=node_ip, source_path=tmp_source, target_ip=node_ip, target_path=tmp_target, ) assert_file(True, tmp_source, "level0.txt") assert_file(True, tmp_target, "level0.txt") assert_file(True, tmp_target, "subdir/level1.txt") assert_file(False, tmp_target, "level0_new.txt") # Add new file with open(os.path.join(tmp_source, "level0_new.txt"), "w") as f: f.write("Data\n") # Modify existing file with open(os.path.join(tmp_source, "subdir", "level1.txt"), "w") as f: f.write("New data\n") unpack, pack_actor, files_stats = _sync_dir_between_different_nodes( source_ip=node_ip, source_path=tmp_source, target_ip=node_ip, target_path=tmp_target, return_futures=True, ) files_stats = ray.get(files_stats) tarball = ray.get(pack_actor.get_full_data.remote()) assert "./level0.txt" in files_stats assert "./level0_new.txt" not in files_stats # Was not in target dir assert "subdir/level1.txt" in files_stats with tarfile.open(fileobj=io.BytesIO(tarball)) as tar: files_in_tar = tar.getnames() assert "./level0.txt" not in files_in_tar assert "./level0_new.txt" in files_in_tar assert "subdir/level1.txt" in files_in_tar assert len(files_in_tar) == 7 # 3 files, 4 dirs (including root) ray.get(unpack) # Wait until finished for teardown
def test_max_size_exceeded(ray_start_2_cpus, temp_data_dirs): tmp_source, tmp_target = temp_data_dirs node_ip = ray.util.get_node_ip_address() with pytest.raises(RayTaskError): _sync_dir_between_different_nodes( source_ip=node_ip, source_path=tmp_source, target_ip=node_ip, target_path=tmp_target, max_size_bytes=2, )
def test_sync_nodes(ray_start_2_cpus, temp_data_dirs): """Check that syncing between nodes works (data is found in target directory)""" tmp_source, tmp_target = temp_data_dirs assert_file(True, tmp_source, "level0.txt") assert_file(True, tmp_source, "subdir/level1.txt") assert_file(False, tmp_target, "level0.txt") assert_file(False, tmp_target, "subdir/level1.txt") node_ip = ray.util.get_node_ip_address() _sync_dir_between_different_nodes( source_ip=node_ip, source_path=tmp_source, target_ip=node_ip, target_path=tmp_target, ) assert_file(True, tmp_target, "level0.txt") assert_file(True, tmp_target, "subdir/level1.txt")
def test_multi_sync_different_node(ray_start_2_cpus, temp_data_dirs, num_workers): """Check that multiple competing syncs to the same dir don't interfere""" tmp_source, tmp_target = temp_data_dirs assert_file(True, tmp_source, "level0.txt") assert_file(True, tmp_source, "subdir/level1.txt") node_ip = ray.util.get_node_ip_address() futures = [ _sync_dir_between_different_nodes( source_ip=node_ip, source_path=tmp_source, target_ip=node_ip, target_path=tmp_target, return_futures=True, )[0] for _ in range(num_workers) ] ray.get(futures) assert_file(True, tmp_target, "level0.txt") assert_file(True, tmp_target, "subdir/level1.txt")
def _testSyncBetweenNodesAndDelete(self, num_workers: int = 1): temp_source = tempfile.mkdtemp() temp_up_target = tempfile.mkdtemp() temp_down_target = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, temp_source) self.addCleanup(shutil.rmtree, temp_up_target, ignore_errors=True) self.addCleanup(shutil.rmtree, temp_down_target) os.makedirs(os.path.join(temp_source, "dir_level0", "dir_level1")) with open(os.path.join(temp_source, "dir_level0", "file_level1.txt"), "w") as f: f.write("Data\n") # Sanity check self._check_dir_contents(temp_source) node_ip = ray.util.get_node_ip_address() futures = [ _sync_dir_between_different_nodes( source_ip=node_ip, source_path=temp_source, target_ip=node_ip, target_path=temp_up_target, return_futures=True, )[0] for i in range(num_workers) ] ray.get(futures) # Check sync up self._check_dir_contents(temp_up_target) # Max size exceeded with self.assertRaises(RayTaskError): _sync_dir_between_different_nodes( source_ip=node_ip, source_path=temp_up_target, target_ip=node_ip, target_path=temp_down_target, max_size_bytes=2, ) assert not os.listdir(temp_down_target) futures = [ _sync_dir_between_different_nodes( source_ip=node_ip, source_path=temp_up_target, target_ip=node_ip, target_path=temp_down_target, return_futures=True, )[0] for i in range(num_workers) ] ray.get(futures) # Check sync down self._check_dir_contents(temp_down_target) # Delete in some dir delete_on_node(node_ip=node_ip, path=temp_up_target) assert not os.path.exists(temp_up_target)
def _sync_function(self, *args, **kwargs): return _sync_dir_between_different_nodes(*args, **kwargs)