def test_num_cpus(): all_cores = psutil.cpu_count(logical=False) returned_cores = systems_utils.get_num_cores(reserved_for_os=0) assert all_cores == returned_cores returned_cores = systems_utils.get_num_cores(reserved_for_os=2) if all_cores <= returned_cores: # CI machines may have few cores. assert all_cores == returned_cores else: assert all_cores - 2 == returned_cores
def get_num_cores(app: ArrayApplication): if isinstance(app.system, RaySystem): system: RaySystem = app.system nodes = system.nodes() return sum(map(lambda n: n["Resources"]["CPU"], nodes)) else: assert isinstance(app.system, SerialSystem) return systems_utils.get_num_cores()
def num_cores_total(self): if isinstance(self.system, RaySystem): system: RaySystem = self.system nodes = system.nodes() num_cores = sum(map(lambda n: n["Resources"]["CPU"], nodes)) else: assert isinstance(self.system, SerialSystem) num_cores = systems_utils.get_num_cores() return int(num_cores)
def __init__( self, use_head: bool = False, num_nodes: Optional[int] = None, num_cpus: Optional[int] = None, ): self._use_head = use_head self._num_nodes = num_nodes self.num_cpus = int(get_num_cores()) if num_cpus is None else num_cpus self._manage_ray = True self._remote_functions = {} self._actors: dict = {} self._actor_node_index = 0 self._available_nodes = [] self._head_node = None self._worker_nodes = [] self._devices: List[DeviceID] = [] self._device_to_node: Dict[DeviceID, Dict] = {}
def get_app(mode): if mode == "serial": system: System = SerialSystem(compute_module=numpy_compute) elif mode.startswith("ray"): ray.init(num_cpus=systems_utils.get_num_cores()) if mode == "ray-task": scheduler: RayScheduler = TaskScheduler(compute_module=numpy_compute, use_head=True) elif mode == "ray-cyclic": cluster_shape = (1, 1) scheduler: RayScheduler = BlockCyclicScheduler(compute_module=numpy_compute, cluster_shape=cluster_shape, use_head=True, verbose=True) else: raise Exception() system: System = RaySystem(compute_module=numpy_compute, scheduler=scheduler) else: raise Exception() system.init() return ArrayApplication(system=system, filesystem=FileSystem(system))
def get_app(system_name, device_grid_name="cyclic"): if system_name == "serial": system: SystemInterface = SerialSystem() elif system_name == "ray": assert not ray.is_initialized() ray.init(num_cpus=systems_utils.get_num_cores()) system: SystemInterface = RaySystem(use_head=True) else: raise Exception("Unexpected system name %s" % system_name) system.init() cluster_shape = (1, 1) if device_grid_name == "cyclic": device_grid: DeviceGrid = CyclicDeviceGrid(cluster_shape, "cpu", system.devices()) elif device_grid_name == "packed": device_grid: DeviceGrid = PackedDeviceGrid(cluster_shape, "cpu", system.devices()) else: raise Exception("Unexpected device grid name %s" % device_grid_name) cm = ComputeManager.create(system, numpy_compute, device_grid) fs = FileSystem(cm) return ArrayApplication(cm, fs)
def test_app_manager(compute_name, system_name, device_grid_name, num_cpus): settings.use_head = True settings.compute_name = compute_name settings.system_name = system_name settings.device_grid_name = device_grid_name settings.num_cpus = num_cpus app: ArrayApplication = application_manager.instance() print(settings.num_cpus, num_cpus, app.cm.num_cores_total()) app_arange = app.arange(0, shape=(10, ), block_shape=(10, )) assert np.allclose(np.arange(10), app_arange.get()) if num_cpus is None: assert app.cm.num_cores_total() == get_num_cores() else: assert app.cm.num_cores_total() == num_cpus application_manager.destroy() assert not application_manager.is_initialized() time.sleep(1) # Revert for other tests. settings.compute_name = "numpy" settings.system_name = "ray" settings.device_grid_name = "cyclic" settings.num_cpus = None
def test_block_shape(nps_app_inst): app = nps_app_inst dtype = np.float64 shape = (10**9, 250) cluster_shape = (1, 1) num_cores = 64 block_shape = app.compute_block_shape(shape=shape, dtype=dtype, cluster_shape=cluster_shape, num_cores=num_cores) grid: ArrayGrid = ArrayGrid(shape, block_shape, dtype.__name__) assert grid.grid_shape == (num_cores, 1) cluster_shape = (16, 1) num_cores = 64 * np.product(cluster_shape) block_shape = app.compute_block_shape(shape=shape, dtype=dtype, cluster_shape=cluster_shape, num_cores=num_cores) grid: ArrayGrid = ArrayGrid(shape, block_shape, dtype.__name__) assert grid.grid_shape == (num_cores, 1) shape = (250, 10**9) cluster_shape = (1, 16) block_shape = app.compute_block_shape(shape=shape, dtype=dtype, cluster_shape=cluster_shape, num_cores=num_cores) grid: ArrayGrid = ArrayGrid(shape, block_shape, dtype.__name__) assert grid.grid_shape == (1, num_cores) shape = (10**4, 10**4) cluster_shape = (1, 1) num_cores = 64 block_shape = app.compute_block_shape(shape=shape, dtype=dtype, cluster_shape=cluster_shape, num_cores=num_cores) grid: ArrayGrid = ArrayGrid(shape, block_shape, dtype.__name__) assert grid.grid_shape == (int(num_cores**.5), int(num_cores**.5)) shape = (10**4, 10**4 // dtype(0).nbytes) block_shape = app.compute_block_shape(shape=shape, dtype=dtype, cluster_shape=cluster_shape, num_cores=num_cores) grid: ArrayGrid = ArrayGrid(shape, block_shape, dtype.__name__) assert grid.grid_shape != (1, 1) shape = (10**4, 10**4 // dtype(0).nbytes - 1) block_shape = app.compute_block_shape(shape=shape, dtype=dtype, cluster_shape=cluster_shape, num_cores=num_cores) grid: ArrayGrid = ArrayGrid(shape, block_shape, dtype.__name__) assert grid.grid_shape == (1, 1) shape = (10**4, 10**4) block_shape = app.compute_block_shape(shape=shape, dtype=dtype, num_cores=num_cores) grid: ArrayGrid = ArrayGrid(shape, block_shape, dtype.__name__) assert grid.grid_shape == (int(num_cores**.5), int(num_cores**.5)) cluster_shape = (12, 1) num_cores = systems_utils.get_num_cores() block_shape = app.compute_block_shape(shape=shape, dtype=dtype, cluster_shape=cluster_shape) grid: ArrayGrid = ArrayGrid(shape, block_shape, dtype.__name__) assert grid.grid_shape == (num_cores, 1)
def test_num_cores(app_inst: ArrayApplication): assert np.allclose(app_inst.cm.num_cores_total(), systems_utils.get_num_cores())
def __init__(self, num_cpus: Optional[int] = None): self.num_cpus = int(get_num_cores()) if num_cpus is None else num_cpus self._remote_functions: dict = {} self._actors: dict = {}
def num_cores_total(self): return int(get_num_cores())