def testEmptyGraph(self, *_): session_id = str(uuid.uuid4()) addr = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(AssignerActor, uid=AssignerActor.default_uid()) resource_ref.set_worker_meta('localhost:12345', dict(hardware=dict(cpu_total=4))) resource_ref.set_worker_meta('localhost:23456', dict(hardware=dict(cpu_total=4))) graph_key = str(uuid.uuid4()) serialized_graph = serialize_graph(DAG()) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid( session_id, graph_key)) graph_ref.execute_graph() self.assertEqual(graph_ref.get_state(), GraphState.SUCCEEDED)
def testErrorOnPrepare(self, *_): session_id = str(uuid.uuid4()) addr = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(AssignerActor, uid=AssignerActor.default_uid()) resource_ref.set_worker_meta('localhost:12345', dict(hardware=dict(cpu_total=4))) resource_ref.set_worker_meta('localhost:23456', dict(hardware=dict(cpu_total=4))) # error occurred in create_operand_actors graph_key = str(uuid.uuid4()) expr = mt.random.random((8, 2), chunk_size=2) + 1 graph = expr.build_graph(compose=False) serialized_graph = serialize_graph(graph) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid(session_id, graph_key)) def _mock_raises(*_, **__): raise RuntimeError with patch_method(GraphActor.create_operand_actors, new=_mock_raises): with self.assertRaises(RuntimeError): graph_ref.execute_graph() self.assertEqual(graph_ref.get_state(), GraphState.FAILED) graph_ref.destroy() # interrupted during create_operand_actors graph_key = str(uuid.uuid4()) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid(session_id, graph_key)) def _mock_cancels(*_, **__): graph_meta_ref = pool.actor_ref(GraphMetaActor.gen_uid(session_id, graph_key)) graph_meta_ref.set_state(GraphState.CANCELLING) with patch_method(GraphActor.create_operand_actors, new=_mock_cancels): graph_ref.execute_graph() self.assertEqual(graph_ref.get_state(), GraphState.CANCELLED) # interrupted during previous steps graph_key = str(uuid.uuid4()) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid(session_id, graph_key)) def _mock_cancels(*_, **__): graph_meta_ref = pool.actor_ref(GraphMetaActor.gen_uid(session_id, graph_key)) graph_meta_ref.set_state(GraphState.CANCELLING) return dict() with patch_method(GraphAnalyzer.calc_operand_assignments, new=_mock_cancels): graph_ref.execute_graph() self.assertEqual(graph_ref.get_state(), GraphState.CANCELLED)
def testAssignerActor(self): mock_scheduler_addr = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=mock_scheduler_addr) as pool: cluster_info_ref = pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) endpoint1 = 'localhost:12345' endpoint2 = 'localhost:23456' res = dict(hardware=dict(cpu=4, memory=4096)) def write_mock_meta(): resource_ref.set_worker_meta(endpoint1, res) resource_ref.set_worker_meta(endpoint2, res) g = gevent.spawn(write_mock_meta) g.join() assigner_ref = pool.create_actor(AssignerActor, uid=AssignerActor.default_uid()) session_id = str(uuid.uuid4()) op_key = str(uuid.uuid4()) chunk_key1 = str(uuid.uuid4()) chunk_key2 = str(uuid.uuid4()) chunk_key3 = str(uuid.uuid4()) op_info = { 'op_name': 'test_op', 'io_meta': dict(input_chunks=[chunk_key1, chunk_key2, chunk_key3]), 'retries': 0, 'optimize': { 'depth': 0, 'demand_depths': (), 'successor_size': 1, 'descendant_size': 0 } } chunk_meta_client = ChunkMetaClient(pool, cluster_info_ref) chunk_meta_client.set_chunk_meta(session_id, chunk_key1, size=512, workers=(endpoint1,)) chunk_meta_client.set_chunk_meta(session_id, chunk_key2, size=512, workers=(endpoint1,)) chunk_meta_client.set_chunk_meta(session_id, chunk_key3, size=512, workers=(endpoint2,)) reply_ref = pool.create_actor(PromiseReplyTestActor) reply_callback = ((reply_ref.uid, reply_ref.address), 'reply') assigner_ref.apply_for_resource(session_id, op_key, op_info, callback=reply_callback) while not reply_ref.get_reply(): gevent.sleep(0.1) _, ret_value = reply_ref.get_reply() self.assertEqual(ret_value[0], endpoint1)
def testAssignerActor(self, *_): mock_scheduler_addr = f'127.0.0.1:{get_next_port()}' with create_actor_pool(n_process=1, backend='gevent', address=mock_scheduler_addr) as pool: cluster_info_ref = pool.create_actor( SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) endpoint1 = 'localhost:12345' endpoint2 = 'localhost:23456' res = dict(hardware=dict(cpu=4, mem_quota=4096)) resource_ref.set_worker_meta(endpoint1, res) resource_ref.set_worker_meta(endpoint2, res) assigner_ref = pool.create_actor(AssignerActor, uid=AssignerActor.default_uid()) session_id = str(uuid.uuid4()) op_key = str(uuid.uuid4()) chunk_key1 = str(uuid.uuid4()) chunk_key2 = str(uuid.uuid4()) chunk_key3 = str(uuid.uuid4()) op_info = { 'op_name': 'test_op', 'io_meta': dict(input_chunks=[chunk_key1, chunk_key2, chunk_key3]), 'retries': 0, 'optimize': { 'depth': 0, 'demand_depths': (), 'successor_size': 1, 'descendant_size': 0 } } chunk_meta_client = ChunkMetaClient(pool, cluster_info_ref) chunk_meta_client.set_chunk_meta(session_id, chunk_key1, size=512, workers=(endpoint1, )) chunk_meta_client.set_chunk_meta(session_id, chunk_key2, size=512, workers=(endpoint1, )) chunk_meta_client.set_chunk_meta(session_id, chunk_key3, size=512, workers=(endpoint2, )) uid = OperandActor.gen_uid(session_id, op_key) reply_ref = pool.create_actor(MockOperandActor, uid=uid) assigner_ref.apply_for_resource(session_id, op_key, op_info) while not reply_ref.get_worker_ep(): gevent.sleep(0.1) self.assertEqual(reply_ref.get_worker_ep(), endpoint1) with self.run_actor_test(pool) as test_actor, self.assertRaises( TimeoutError): assigner_p_ref = test_actor.promise_ref(assigner_ref) try: options.scheduler.assign_timeout = 1 res = dict(hardware=dict(cpu=4, mem_quota=0)) resource_ref.set_worker_meta(endpoint1, res) resource_ref.set_worker_meta(endpoint2, res) self.waitp( assigner_p_ref.apply_for_resource(session_id, op_key, op_info, _promise=True)) finally: options.scheduler.assign_timeout = 600
def prepare_graph_in_pool(self, expr, clean_io_meta=True, compose=False): session_id = str(uuid.uuid4()) graph_key = str(uuid.uuid4()) graph = expr.build_graph(compose=compose) serialized_graph = serialize_graph(graph) chunked_graph = expr.build_graph(compose=compose, tiled=True) addr = '127.0.0.1:%d' % get_next_port() with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool: pool.create_actor(SchedulerClusterInfoActor, [pool.cluster_info.address], uid=SchedulerClusterInfoActor.default_uid()) resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_uid()) pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_uid()) pool.create_actor(AssignerActor, uid=AssignerActor.default_uid()) graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph, uid=GraphActor.gen_uid(session_id, graph_key)) graph_ref.prepare_graph(compose=compose) fetched_graph = graph_ref.get_chunk_graph() self.assertIsNotNone(fetched_graph) self.assertEqual(len(chunked_graph), len(fetched_graph)) graph_ref.analyze_graph(do_placement=False) op_infos = graph_ref.get_operand_info() for n in fetched_graph: depth = op_infos[n.op.key]['optimize']['depth'] self.assertIsNotNone(depth) successor_size = op_infos[n.op.key]['optimize']['successor_size'] self.assertIsNotNone(successor_size) descendant_size = op_infos[n.op.key]['optimize']['descendant_size'] self.assertIsNotNone(descendant_size) resource_ref.set_worker_meta('localhost:12345', dict(hardware=dict(cpu_total=4))) resource_ref.set_worker_meta('localhost:23456', dict(hardware=dict(cpu_total=4))) graph_ref.analyze_graph() op_infos = graph_ref.get_operand_info() for n in fetched_graph: if fetched_graph.count_predecessors(n) != 0: continue target_worker = op_infos[n.op.key]['target_worker'] self.assertIsNotNone(target_worker) graph_ref.create_operand_actors(_clean_info=clean_io_meta) op_infos = graph_ref.get_operand_info() if not clean_io_meta: orig_metas = dict() for n in fetched_graph: try: meta = orig_metas[n.op.key] except KeyError: meta = orig_metas[n.op.key] = dict( predecessors=set(), successors=set(), input_chunks=set(), chunks=set() ) meta['predecessors'].update([pn.op.key for pn in fetched_graph.iter_predecessors(n)]) meta['successors'].update([sn.op.key for sn in fetched_graph.iter_successors(n)]) meta['input_chunks'].update([pn.key for pn in fetched_graph.iter_predecessors(n)]) meta['chunks'].update([c.key for c in n.op.outputs]) for n in fetched_graph: self.assertEqual(op_infos[n.op.key]['op_name'], type(n.op).__name__) io_meta = op_infos[n.op.key]['io_meta'] orig_io_meta = orig_metas[n.op.key] self.assertSetEqual(set(io_meta['predecessors']), set(orig_io_meta['predecessors'])) self.assertSetEqual(set(io_meta['successors']), set(orig_io_meta['successors'])) self.assertSetEqual(set(io_meta['input_chunks']), set(orig_io_meta['input_chunks'])) self.assertSetEqual(set(io_meta['chunks']), set(orig_io_meta['chunks'])) yield pool, graph_ref