def testEmptyGraph(self, *_): session_id = str(uuid.uuid4()) addr = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(AssignerActor, uid=AssignerActor.gen_uid(session_id)) resource_ref.set_worker_meta('localhost:12345', dict(hardware=dict(cpu_total=4))) resource_ref.set_worker_meta('localhost:23456', dict(hardware=dict(cpu_total=4))) graph_key = str(uuid.uuid4()) serialized_graph = serialize_graph(DAG()) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid( session_id, graph_key)) graph_ref.execute_graph() self.assertEqual(graph_ref.get_state(), GraphState.SUCCEEDED)
def _prepare_test_graph(self, session_id, graph_key, mock_workers): addr = f'127.0.0.1:{get_next_port()}' a1 = mt.random.random((100,)) a2 = mt.random.random((100,)) s = a1 + a2 v1, v2 = mt.split(s, 2) graph = TileableGraph([v1.data, v2.data]) builder = TileableGraphBuilder(graph) next(iter(builder.build())) with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(AssignerActor, uid=AssignerActor.gen_uid(session_id)) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialize_graph(graph), uid=GraphActor.gen_uid(session_id, graph_key)) for w in mock_workers: resource_ref.set_worker_meta(w, dict(hardware=dict(cpu=4, cpu_total=4, memory=1600))) graph_ref.prepare_graph() graph_ref.analyze_graph() graph_ref.create_operand_actors(_start=False) yield pool, graph_ref
def start_transfer_test_pool(**kwargs): address = kwargs.pop('address') plasma_size = kwargs.pop('plasma_size') with create_actor_pool(n_process=1, backend='gevent', address=address, **kwargs) as pool: pool.create_actor(SchedulerClusterInfoActor, schedulers=[address], uid=SchedulerClusterInfoActor.default_uid()) pool.create_actor(WorkerClusterInfoActor, schedulers=[address], uid=WorkerClusterInfoActor.default_uid()) pool.create_actor(PlasmaKeyMapActor, uid=PlasmaKeyMapActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(DispatchActor, uid=DispatchActor.default_uid()) pool.create_actor(QuotaActor, 1024 * 1024 * 20, uid=MemQuotaActor.default_uid()) chunk_holder_ref = pool.create_actor( ChunkHolderActor, plasma_size, uid=ChunkHolderActor.default_uid()) pool.create_actor(SpillActor) pool.create_actor(StatusActor, address, uid=StatusActor.default_uid()) yield pool chunk_holder_ref.destroy()
def _start_calc_pool(self): mock_addr = f'127.0.0.1:{get_next_port()}' with self.create_pool(n_process=1, backend='gevent', address=mock_addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [mock_addr], uid=SchedulerClusterInfoActor.default_uid()) pool.create_actor(WorkerClusterInfoActor, [mock_addr], uid=WorkerClusterInfoActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(StatusActor, mock_addr, uid=StatusActor.default_uid()) pool.create_actor(PlasmaKeyMapActor, uid=PlasmaKeyMapActor.default_uid()) pool.create_actor(WorkerDaemonActor, uid=WorkerDaemonActor.default_uid()) pool.create_actor(DispatchActor, uid=DispatchActor.default_uid()) pool.create_actor(StorageManagerActor, uid=StorageManagerActor.default_uid()) pool.create_actor(IORunnerActor) pool.create_actor(QuotaActor, 1024**2, uid=MemQuotaActor.default_uid()) shared_holder_ref = pool.create_actor( SharedHolderActor, uid=SharedHolderActor.default_uid()) pool.create_actor(InProcHolderActor) pool.create_actor(CpuCalcActor, uid=CpuCalcActor.default_uid()) with self.run_actor_test(pool) as test_actor: try: yield pool, test_actor finally: shared_holder_ref.destroy()
def start_transfer_test_pool(**kwargs): address = kwargs.pop('address') plasma_size = kwargs.pop('plasma_size') with create_actor_pool(n_process=1, backend='gevent', address=address, **kwargs) as pool: pool.create_actor(SchedulerClusterInfoActor, [address], uid=SchedulerClusterInfoActor.default_uid()) pool.create_actor(WorkerClusterInfoActor, [address], uid=WorkerClusterInfoActor.default_uid()) pool.create_actor(PlasmaKeyMapActor, uid=PlasmaKeyMapActor.default_uid()) pool.create_actor(StorageManagerActor, uid=StorageManagerActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(DispatchActor, uid=DispatchActor.default_uid()) pool.create_actor(QuotaActor, 1024 * 1024 * 20, uid=MemQuotaActor.default_uid()) shared_holder_ref = pool.create_actor(SharedHolderActor, plasma_size, uid=SharedHolderActor.default_uid()) pool.create_actor(StatusActor, address, uid=StatusActor.default_uid()) pool.create_actor(IORunnerActor) pool.create_actor(StorageClientActor, uid=StorageClientActor.default_uid()) pool.create_actor(InProcHolderActor) pool.create_actor(ReceiverManagerActor, uid=ReceiverManagerActor.default_uid()) try: yield pool finally: shared_holder_ref.destroy()
def testGetTensorNsplits(self, *_): session_id = 'mock_session_id' graph_key = 'mock_graph_key' tensor_key = 'mock_tensor_key' serialized_graph = 'mock_serialized_graph' graph_uid = GraphActor.gen_uid(session_id, graph_key) self.pool.create_actor(GraphActor, session_id, serialized_graph, graph_key, uid=graph_uid) self.pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) mock_indexes = [ (((3, 4, 5, 6),), OrderedDict(zip([(0, ), (1,), (2,), (3,)], ['chunk_key1', 'chunk_key2', 'chunk_key3', 'chunk_key4']))), (((3, 2), (4, 2)), OrderedDict(zip([(0, 0), (0, 1), (1, 0), (1, 1)], ['chunk_key1', 'chunk_key2', 'chunk_key3', 'chunk_key4']))) ] mock_shapes = [ [(3,), (4,), (5,), (6,)], [(3, 4), (3, 2), (2, 4), (2, 2)] ] GraphActor.get_tileable_meta.side_effect = mock_indexes ChunkMetaClient.batch_get_chunk_shape.side_effect = mock_shapes nsplits = self.api.get_tileable_nsplits(session_id, graph_key, tensor_key) self.assertEqual(((3, 4, 5, 6),), nsplits) nsplits = self.api.get_tileable_nsplits(session_id, graph_key, tensor_key) self.assertEqual(((3, 2), (4, 2)), nsplits)
def testFailoverMessage(self): mock_session_id = str(uuid.uuid4()) mock_graph_key = str(uuid.uuid4()) mock_chunk_key = str(uuid.uuid4()) addr = '127.0.0.1:%d' % get_next_port() mock_worker_addr = '127.0.0.1:54132' options.scheduler.worker_blacklist_time = 0.5 with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool: cluster_info_ref = pool.create_actor( SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) session_manager_ref = pool.create_actor( SessionManagerActor, uid=SessionManagerActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) session_ref = pool.actor_ref( session_manager_ref.create_session(mock_session_id)) chunk_meta_client = ChunkMetaClient(pool, cluster_info_ref) chunk_meta_client.set_chunk_meta(mock_session_id, mock_chunk_key, size=80, shape=(10, ), workers=(mock_worker_addr, )) with mock.patch(GraphActor.__module__ + '.' + GraphActor.__name__, new=MockGraphActor): session_ref.submit_tileable_graph(None, mock_graph_key) graph_ref = pool.actor_ref( GraphActor.gen_uid(mock_session_id, mock_graph_key)) expire_time = time.time( ) - options.scheduler.status_timeout - 1 resource_ref.set_worker_meta(mock_worker_addr, dict(update_time=expire_time)) resource_ref.detect_dead_workers(_tell=True) pool.sleep(0.2) _, removes, lost_chunks = graph_ref.get_worker_change_args() self.assertListEqual(removes, [mock_worker_addr]) self.assertListEqual(lost_chunks, [mock_chunk_key]) self.assertNotIn(mock_worker_addr, resource_ref.get_workers_meta()) resource_ref.set_worker_meta(mock_worker_addr, dict(update_time=time.time())) self.assertNotIn(mock_worker_addr, resource_ref.get_workers_meta()) pool.sleep(0.4) resource_ref.set_worker_meta(mock_worker_addr, dict(update_time=time.time())) self.assertIn(mock_worker_addr, resource_ref.get_workers_meta())
def testErrorOnPrepare(self, *_): session_id = str(uuid.uuid4()) addr = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(AssignerActor, uid=AssignerActor.default_uid()) resource_ref.set_worker_meta('localhost:12345', dict(hardware=dict(cpu_total=4))) resource_ref.set_worker_meta('localhost:23456', dict(hardware=dict(cpu_total=4))) # error occurred in create_operand_actors graph_key = str(uuid.uuid4()) expr = mt.random.random((8, 2), chunk_size=2) + 1 graph = expr.build_graph(compose=False) serialized_graph = serialize_graph(graph) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid(session_id, graph_key)) def _mock_raises(*_, **__): raise RuntimeError with patch_method(GraphActor.create_operand_actors, new=_mock_raises): with self.assertRaises(RuntimeError): graph_ref.execute_graph() self.assertEqual(graph_ref.get_state(), GraphState.FAILED) graph_ref.destroy() # interrupted during create_operand_actors graph_key = str(uuid.uuid4()) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid(session_id, graph_key)) def _mock_cancels(*_, **__): graph_meta_ref = pool.actor_ref(GraphMetaActor.gen_uid(session_id, graph_key)) graph_meta_ref.set_state(GraphState.CANCELLING) with patch_method(GraphActor.create_operand_actors, new=_mock_cancels): graph_ref.execute_graph() self.assertEqual(graph_ref.get_state(), GraphState.CANCELLED) # interrupted during previous steps graph_key = str(uuid.uuid4()) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid(session_id, graph_key)) def _mock_cancels(*_, **__): graph_meta_ref = pool.actor_ref(GraphMetaActor.gen_uid(session_id, graph_key)) graph_meta_ref.set_state(GraphState.CANCELLING) return dict() with patch_method(GraphAnalyzer.calc_operand_assignments, new=_mock_cancels): graph_ref.execute_graph() self.assertEqual(graph_ref.get_state(), GraphState.CANCELLED)
def testOperandActorWithCancel(self, *_): arr = mt.random.randint(10, size=(10, 8), chunk_size=4) arr_add = mt.random.randint(10, size=(10, 8), chunk_size=4) arr2 = arr + arr_add session_id = str(uuid.uuid4()) graph_key = str(uuid.uuid4()) graph = arr2.build_graph(compose=False) with create_actor_pool(n_process=1, backend='gevent') as pool: pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(AssignerActor, uid=AssignerActor.gen_uid(session_id)) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialize_graph(graph), uid=GraphActor.gen_uid(session_id, graph_key)) def _build_mock_ref(uid=None, address=None): try: return pool.create_actor( FakeExecutionActor, exec_delay=0.2, uid=FakeExecutionActor.gen_uid(address)) except ActorAlreadyExist: return pool.actor_ref(FakeExecutionActor.gen_uid(address)) # handle mock objects OperandActor._get_raw_execution_ref.side_effect = _build_mock_ref mock_resource = dict(hardware=dict(cpu=4, cpu_total=4, memory=512)) for idx in range(20): resource_ref.set_worker_meta('localhost:%d' % (idx + 12345), mock_resource) graph_ref.prepare_graph(compose=False) fetched_graph = graph_ref.get_chunk_graph() graph_ref.analyze_graph() final_keys = set() for c in fetched_graph: if fetched_graph.count_successors(c) == 0: final_keys.add(c.op.key) graph_ref.create_operand_actors() graph_meta_ref = pool.actor_ref(GraphMetaActor.gen_uid(session_id, graph_key)) start_time = time.time() cancel_called = False while True: pool.sleep(0.05) if not cancel_called and time.time() > start_time + 0.3: cancel_called = True graph_ref.stop_graph(_tell=True) if time.time() - start_time > 30: raise SystemError('Wait for execution finish timeout') if graph_meta_ref.get_state() in (GraphState.SUCCEEDED, GraphState.FAILED, GraphState.CANCELLED): break self.assertEqual(graph_meta_ref.get_state(), GraphState.CANCELLED)
def _run_operand_case(session_id, graph_key, tensor, execution_creator): graph = tensor.build_graph(compose=False) with create_actor_pool(n_process=1, backend='gevent') as pool: pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(AssignerActor, uid=AssignerActor.gen_uid(session_id)) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialize_graph(graph), uid=GraphActor.gen_uid( session_id, graph_key)) def _build_mock_ref(uid=None, address=None): try: return execution_creator( pool, FakeExecutionActor.gen_uid(address)) except ActorAlreadyExist: return pool.actor_ref(FakeExecutionActor.gen_uid(address)) # handle mock objects OperandActor._get_raw_execution_ref.side_effect = _build_mock_ref mock_resource = dict(hardware=dict(cpu=4, cpu_total=4, memory=512)) resource_ref.set_worker_meta('localhost:12345', mock_resource) resource_ref.set_worker_meta('localhost:23456', mock_resource) graph_ref.prepare_graph() fetched_graph = graph_ref.get_chunk_graph() graph_ref.analyze_graph() final_keys = set() for c in fetched_graph: if fetched_graph.count_successors(c) == 0: final_keys.add(c.op.key) graph_ref.create_operand_actors() graph_meta_ref = pool.actor_ref( GraphMetaActor.gen_uid(session_id, graph_key)) start_time = time.time() while True: pool.sleep(0.1) if time.time() - start_time > 30: raise SystemError('Wait for execution finish timeout') if graph_meta_ref.get_state() in (GraphState.SUCCEEDED, GraphState.FAILED, GraphState.CANCELLED): break
def _start_worker_process(self, cuda=False, cuda_device=None, extra_env=None, modules=None, check_timeout=None): mock_scheduler_addr = f'127.0.0.1:{get_next_port()}' try: with create_actor_pool(n_process=1, backend='gevent', address=mock_scheduler_addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [mock_scheduler_addr], uid=SchedulerClusterInfoActor.default_uid()) pool.create_actor(SessionManagerActor, uid=SessionManagerActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) resource_ref = pool.create_actor( ResourceActor, uid=ResourceActor.default_uid()) args = [ sys.executable, '-m', 'mars.worker', '-a', '127.0.0.1', '--schedulers', mock_scheduler_addr, '--cpu-procs', '1', '--cache-mem', '10m', '--spill-dir', self._spill_dir, '--log-level', 'debug', '--log-format', '%(asctime)-15s %(message)s', '--ignore-avail-mem' ] if modules: args.extend(['--load-modules', ','.join(modules)]) env = os.environ.copy() env.update(extra_env or dict()) if cuda: env['CUDA_VISIBLE_DEVICES'] = cuda_device proc = subprocess.Popen(args, env=env) worker_endpoint = self._wait_worker_ready( proc, resource_ref, timeout=check_timeout) yield pool, worker_endpoint finally: if proc.poll() is None: proc.send_signal(signal.SIGINT) check_time = time.time() while True: time.sleep(0.1) if proc.poll( ) is not None or time.time() - check_time >= 5: break if proc.poll() is None: proc.kill() if os.path.exists(options.worker.plasma_socket): os.unlink(options.worker.plasma_socket)
def testAssignerActor(self): mock_scheduler_addr = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=mock_scheduler_addr) as pool: cluster_info_ref = pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) endpoint1 = 'localhost:12345' endpoint2 = 'localhost:23456' res = dict(hardware=dict(cpu=4, memory=4096)) def write_mock_meta(): resource_ref.set_worker_meta(endpoint1, res) resource_ref.set_worker_meta(endpoint2, res) g = gevent.spawn(write_mock_meta) g.join() assigner_ref = pool.create_actor(AssignerActor, uid=AssignerActor.default_uid()) session_id = str(uuid.uuid4()) op_key = str(uuid.uuid4()) chunk_key1 = str(uuid.uuid4()) chunk_key2 = str(uuid.uuid4()) chunk_key3 = str(uuid.uuid4()) op_info = { 'op_name': 'test_op', 'io_meta': dict(input_chunks=[chunk_key1, chunk_key2, chunk_key3]), 'retries': 0, 'optimize': { 'depth': 0, 'demand_depths': (), 'successor_size': 1, 'descendant_size': 0 } } chunk_meta_client = ChunkMetaClient(pool, cluster_info_ref) chunk_meta_client.set_chunk_meta(session_id, chunk_key1, size=512, workers=(endpoint1,)) chunk_meta_client.set_chunk_meta(session_id, chunk_key2, size=512, workers=(endpoint1,)) chunk_meta_client.set_chunk_meta(session_id, chunk_key3, size=512, workers=(endpoint2,)) reply_ref = pool.create_actor(PromiseReplyTestActor) reply_callback = ((reply_ref.uid, reply_ref.address), 'reply') assigner_ref.apply_for_resource(session_id, op_key, op_info, callback=reply_callback) while not reply_ref.get_reply(): gevent.sleep(0.1) _, ret_value = reply_ref.get_reply() self.assertEqual(ret_value[0], endpoint1)
def testWorkerProcessRestart(self): mock_scheduler_addr = '127.0.0.1:%d' % get_next_port() try: with create_actor_pool(n_process=1, backend='gevent', address=mock_scheduler_addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [mock_scheduler_addr], uid=SchedulerClusterInfoActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) resource_ref = pool.create_actor( ResourceActor, uid=ResourceActor.default_uid()) proc = subprocess.Popen([ sys.executable, '-m', 'mars.worker', '-a', '127.0.0.1', '--schedulers', mock_scheduler_addr, '--cpu-procs', '1', '--cache-mem', '10m', '--spill-dir', self._spill_dir, '--ignore-avail-mem' ]) worker_endpoint = self._wait_worker_ready(proc, resource_ref) daemon_ref = pool.actor_ref(WorkerDaemonActor.default_uid(), address=worker_endpoint) dispatch_ref = pool.actor_ref(DispatchActor.default_uid(), address=worker_endpoint) cpu_slots = dispatch_ref.get_slots('cpu') calc_ref = pool.actor_ref(cpu_slots[0], address=worker_endpoint) daemon_ref.kill_actor_process(calc_ref) check_start = time.time() while not daemon_ref.is_actor_process_alive(calc_ref): gevent.sleep(0.1) if time.time() - check_start > 10: raise TimeoutError('Check process restart timeout') finally: if proc.poll() is None: proc.send_signal(signal.SIGINT) check_time = time.time() while True: time.sleep(0.1) if proc.poll( ) is not None or time.time() - check_time >= 5: break if proc.poll() is None: proc.kill() if os.path.exists(options.worker.plasma_socket): os.unlink(options.worker.plasma_socket)
def testExecuteWorker(self): mock_scheduler_addr = '127.0.0.1:%d' % get_next_port() try: with create_actor_pool(n_process=1, backend='gevent', address=mock_scheduler_addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [mock_scheduler_addr], uid=SchedulerClusterInfoActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) resource_ref = pool.create_actor( ResourceActor, uid=ResourceActor.default_uid()) proc = subprocess.Popen([ sys.executable, '-m', 'mars.worker', '-a', '127.0.0.1', '--schedulers', mock_scheduler_addr, '--cpu-procs', '1', '--cache-mem', '10m', '--spill-dir', self._spill_dir, '--ignore-avail-mem' ]) worker_endpoint = self._wait_worker_ready(proc, resource_ref) test_ref = pool.create_actor(WorkerProcessTestActor) test_ref.run_test(worker_endpoint, _tell=True) check_time = time.time() while not test_ref.get_reply(): gevent.sleep(0.1) if time.time() - check_time > 20: raise TimeoutError('Check reply timeout') finally: if proc.poll() is None: proc.send_signal(signal.SIGINT) check_time = time.time() while True: time.sleep(0.1) if proc.poll( ) is not None or time.time() - check_time >= 5: break if proc.poll() is None: proc.kill() if os.path.exists(options.worker.plasma_socket): os.unlink(options.worker.plasma_socket)
def _start_worker_process(self, no_cuda=True, cuda_device=None): mock_scheduler_addr = '127.0.0.1:%d' % get_next_port() try: with create_actor_pool(n_process=1, backend='gevent', address=mock_scheduler_addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [mock_scheduler_addr], uid=SchedulerClusterInfoActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) resource_ref = pool.create_actor( ResourceActor, uid=ResourceActor.default_uid()) args = [ sys.executable, '-m', 'mars.worker', '-a', '127.0.0.1', '--schedulers', mock_scheduler_addr, '--cpu-procs', '1', '--cache-mem', '10m', '--spill-dir', self._spill_dir, '--ignore-avail-mem' ] env = os.environ.copy() if no_cuda: args.append('--no-cuda') else: env['CUDA_VISIBLE_DEVICES'] = cuda_device proc = subprocess.Popen(args, env=env) worker_endpoint = self._wait_worker_ready(proc, resource_ref) yield pool, worker_endpoint finally: if proc.poll() is None: proc.send_signal(signal.SIGINT) check_time = time.time() while True: time.sleep(0.1) if proc.poll( ) is not None or time.time() - check_time >= 5: break if proc.poll() is None: proc.kill() if os.path.exists(options.worker.plasma_socket): os.unlink(options.worker.plasma_socket)
def create_standard_actors(cls, pool, address, quota_size=None, with_daemon=True, with_status=True, with_resource=False): quota_size = quota_size or (1024 * 1024) pool.create_actor(SchedulerClusterInfoActor, [address], uid=SchedulerClusterInfoActor.default_uid()) pool.create_actor(WorkerClusterInfoActor, [address], uid=WorkerClusterInfoActor.default_uid()) pool.create_actor(PlasmaKeyMapActor, uid=PlasmaKeyMapActor.default_uid()) if with_resource: pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) if with_daemon: pool.create_actor(WorkerDaemonActor, uid=WorkerDaemonActor.default_uid()) if with_status: pool.create_actor(StatusActor, address, uid=StatusActor.default_uid()) pool.create_actor( ChunkHolderActor, cls.plasma_storage_size, uid=ChunkHolderActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(DispatchActor, uid=DispatchActor.default_uid()) pool.create_actor(QuotaActor, quota_size, uid=MemQuotaActor.default_uid()) pool.create_actor(ExecutionActor, uid=ExecutionActor.default_uid())
def testAssignerActor(self, *_): mock_scheduler_addr = f'127.0.0.1:{get_next_port()}' with create_actor_pool(n_process=1, backend='gevent', address=mock_scheduler_addr) as pool: cluster_info_ref = pool.create_actor( SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) endpoint1 = 'localhost:12345' endpoint2 = 'localhost:23456' res = dict(hardware=dict(cpu=4, mem_quota=4096)) resource_ref.set_worker_meta(endpoint1, res) resource_ref.set_worker_meta(endpoint2, res) assigner_ref = pool.create_actor(AssignerActor, uid=AssignerActor.default_uid()) session_id = str(uuid.uuid4()) op_key = str(uuid.uuid4()) chunk_key1 = str(uuid.uuid4()) chunk_key2 = str(uuid.uuid4()) chunk_key3 = str(uuid.uuid4()) op_info = { 'op_name': 'test_op', 'io_meta': dict(input_chunks=[chunk_key1, chunk_key2, chunk_key3]), 'retries': 0, 'optimize': { 'depth': 0, 'demand_depths': (), 'successor_size': 1, 'descendant_size': 0 } } chunk_meta_client = ChunkMetaClient(pool, cluster_info_ref) chunk_meta_client.set_chunk_meta(session_id, chunk_key1, size=512, workers=(endpoint1, )) chunk_meta_client.set_chunk_meta(session_id, chunk_key2, size=512, workers=(endpoint1, )) chunk_meta_client.set_chunk_meta(session_id, chunk_key3, size=512, workers=(endpoint2, )) uid = OperandActor.gen_uid(session_id, op_key) reply_ref = pool.create_actor(MockOperandActor, uid=uid) assigner_ref.apply_for_resource(session_id, op_key, op_info) while not reply_ref.get_worker_ep(): gevent.sleep(0.1) self.assertEqual(reply_ref.get_worker_ep(), endpoint1) with self.run_actor_test(pool) as test_actor, self.assertRaises( TimeoutError): assigner_p_ref = test_actor.promise_ref(assigner_ref) try: options.scheduler.assign_timeout = 1 res = dict(hardware=dict(cpu=4, mem_quota=0)) resource_ref.set_worker_meta(endpoint1, res) resource_ref.set_worker_meta(endpoint2, res) self.waitp( assigner_p_ref.apply_for_resource(session_id, op_key, op_info, _promise=True)) finally: options.scheduler.assign_timeout = 600
def prepare_graph_in_pool(self, expr, clean_io_meta=True, compose=False): session_id = str(uuid.uuid4()) graph_key = str(uuid.uuid4()) graph = expr.build_graph(compose=compose) serialized_graph = serialize_graph(graph) chunked_graph = expr.build_graph(compose=compose, tiled=True) addr = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(AssignerActor, uid=AssignerActor.gen_uid(session_id)) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid( session_id, graph_key)) graph_ref.prepare_graph(compose=compose) fetched_graph = graph_ref.get_chunk_graph() self.assertIsNotNone(fetched_graph) self.assertEqual(len(chunked_graph), len(fetched_graph)) graph_ref.analyze_graph(do_placement=False) op_infos = graph_ref.get_operand_info() for n in fetched_graph: depth = op_infos[n.op.key]['optimize']['depth'] self.assertIsNotNone(depth) successor_size = op_infos[ n.op.key]['optimize']['successor_size'] self.assertIsNotNone(successor_size) descendant_size = op_infos[ n.op.key]['optimize']['descendant_size'] self.assertIsNotNone(descendant_size) resource_ref.set_worker_meta('localhost:12345', dict(hardware=dict(cpu_total=4))) resource_ref.set_worker_meta('localhost:23456', dict(hardware=dict(cpu_total=4))) graph_ref.analyze_graph() op_infos = graph_ref.get_operand_info() for n in fetched_graph: if fetched_graph.count_predecessors(n) != 0: continue target_worker = op_infos[n.op.key]['target_worker'] self.assertIsNotNone(target_worker) graph_ref.create_operand_actors(_clean_info=clean_io_meta) op_infos = graph_ref.get_operand_info() if not clean_io_meta: orig_metas = dict() for n in fetched_graph: try: meta = orig_metas[n.op.key] except KeyError: meta = orig_metas[n.op.key] = dict(predecessors=set(), successors=set(), input_chunks=set(), chunks=set()) meta['predecessors'].update([ pn.op.key for pn in fetched_graph.iter_predecessors(n) ]) meta['successors'].update( [sn.op.key for sn in fetched_graph.iter_successors(n)]) meta['input_chunks'].update( [pn.key for pn in fetched_graph.iter_predecessors(n)]) meta['chunks'].update([c.key for c in n.op.outputs]) for n in fetched_graph: self.assertEqual(op_infos[n.op.key]['op_name'], type(n.op).__name__) io_meta = op_infos[n.op.key]['io_meta'] orig_io_meta = orig_metas[n.op.key] self.assertSetEqual(set(io_meta['predecessors']), set(orig_io_meta['predecessors'])) self.assertSetEqual(set(io_meta['successors']), set(orig_io_meta['successors'])) self.assertSetEqual(set(io_meta['input_chunks']), set(orig_io_meta['input_chunks'])) self.assertSetEqual(set(io_meta['chunks']), set(orig_io_meta['chunks'])) yield pool, graph_ref