def test_wfp_start_processor(): p = Pipeline() s = Stage() t = Task() t.executable = ['/bin/date'] s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=False) assert wfp.start_processor() assert not wfp._enqueue_thread assert not wfp._dequeue_thread assert not wfp._enqueue_thread_terminate.is_set() assert not wfp._dequeue_thread_terminate.is_set() assert not wfp._wfp_terminate.is_set() assert wfp._wfp_process.is_alive() wfp._wfp_terminate.set() wfp._wfp_process.join()
def test_wfp_initialization(s, b, l): p = Pipeline() stage = Stage() t = Task() t.executable = '/bin/date' stage.add_tasks(t) p.add_stages(stage) rmq_conn_params = pika.ConnectionParameters(host=hostname, port=port) wfp = WFprocessor(sid='rp.session.local.0000', workflow=set([p]), pending_queue=['pending'], completed_queue=['completed'], rmq_conn_params=rmq_conn_params, resubmit_failed=True) assert len(wfp._uid.split('.')) == 2 assert 'wfprocessor' == wfp._uid.split('.')[0] assert wfp._pending_queue == ['pending'] assert wfp._completed_queue == ['completed'] assert wfp._rmq_conn_params == rmq_conn_params assert wfp._wfp_process is None assert wfp._workflow == set([p]) if not isinstance(s, str): wfp = WFprocessor(sid=s, workflow=set([p]), pending_queue=l, completed_queue=l, rmq_conn_params=rmq_conn_params, resubmit_failed=b)
def test_wfp_workflow_incomplete(self, mocked_init, mocked_Logger): wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._logger = mocked_Logger pipe = mock.Mock() pipe.lock = mt.Lock() pipe.completed = False wfp._workflow = set([pipe]) self.assertTrue(wfp.workflow_incomplete()) pipe.completed = True self.assertFalse(wfp.workflow_incomplete()) wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) with self.assertRaises(Exception): wfp.workflow_incomplete()
def test_check_processor(self, mocked_init): wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._enqueue_thread = None wfp._dequeue_thread = None self.assertFalse(wfp.check_processor()) wfp._enqueue_thread = mock.Mock() wfp._enqueue_thread.is_alive = mock.MagicMock( side_effect=[False, False, True, True]) wfp._dequeue_thread = mock.Mock() wfp._dequeue_thread.is_alive = mock.MagicMock( side_effect=[False, True, False, True]) self.assertFalse(wfp.check_processor()) self.assertFalse(wfp.check_processor()) self.assertFalse(wfp.check_processor()) self.assertTrue(wfp.check_processor())
def test_create_workload(self, mocked_init, mocked_advance, mocked_Logger, mocked_Reporter): wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._resubmit_failed = False pipe = mock.Mock() pipe.lock = mt.Lock() pipe.state = states.INITIAL pipe.completed = False pipe.current_stage = 1 stage = mock.Mock() stage.uid = 'stage.0000' stage.state = states.SCHEDULING task = mock.Mock() task.uid = 'task.0000' task.state = states.INITIAL stage.tasks = [task] pipe.stages = [stage] wfp._workflow = set([pipe]) workload, scheduled_stages = wfp._create_workload() self.assertEqual(workload, [task]) self.assertEqual(scheduled_stages, [stage])
def test_wfp_initialization(s, i, b, l): p = Pipeline() st = Stage() t = Task() t.executable = ['/bin/date'] st.add_tasks(t) p.add_stages(st) wfp = WFprocessor(sid='rp.session.local.0000', workflow=set([p]), pending_queue=['pending'], completed_queue=['completed'], mq_hostname=hostname, port=port, resubmit_failed=True) assert len(wfp._uid.split('.')) == 2 assert 'wfprocessor' == wfp._uid.split('.')[0] assert wfp._pending_queue == ['pending'] assert wfp._completed_queue == ['completed'] assert wfp._mq_hostname == hostname assert wfp._port == port assert wfp._wfp_process == None assert wfp._workflow == set([p]) if not isinstance(s, unicode): wfp = WFprocessor(sid=s, workflow=set([p]), pending_queue=l, completed_queue=l, mq_hostname=s, port=i, resubmit_failed=b)
def test_wfp_workflow_incomplete(): p = Pipeline() s = Stage() t = Task() t.executable = ['/bin/date'] s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=False) wfp._initialize_workflow() assert wfp.workflow_incomplete() amgr.workflow = [p] profiler = ru.Profiler(name='radical.entk.temp') p.stages[0].state == states.SCHEDULING p.state == states.SCHEDULED for t in p.stages[0].tasks: t.state = states.COMPLETED import json import pika task_as_dict = json.dumps(t.to_dict()) mq_connection = pika.BlockingConnection(pika.ConnectionParameters(host=amgr._mq_hostname, port=amgr._port)) mq_channel = mq_connection.channel() mq_channel.basic_publish(exchange='', routing_key='%s-completedq-1' % amgr._sid, body=task_as_dict) amgr._terminate_sync = Event() sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread') sync_thread.start() proc = Process(target=func_for_dequeue_test, name='temp-proc', args=(wfp,)) proc.start() proc.join() amgr._terminate_sync.set() sync_thread.join() assert not wfp.workflow_incomplete()
def test_workflow(self, mocked_init, mocked_Logger): wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._workflow = 'test_workflow' self.assertEqual(wfp.workflow, 'test_workflow')
def test_execute_post_exec(self, mocked_init, mocked_Logger, mocked_Profiler): global_advs = set() def _advance_side_effect(obj, obj_type, state): nonlocal global_advs global_advs.add((obj, obj_type, state)) wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._uid = 'wfp.0000' wfp._logger = mocked_Logger wfp._prof = mocked_Profiler wfp._advance = mock.MagicMock(side_effect=_advance_side_effect) pipe = mock.Mock() pipe.lock = mt.Lock() pipe.state = states.INITIAL pipe.uid = 'pipe.0000' pipe.completed = False pipe._increment_stage = mock.MagicMock(return_value=True) pipe.current_stage = 1 pipe2 = mock.Mock() pipe2.lock = mt.Lock() pipe2.state = states.INITIAL pipe2.uid = 'pipe.0001' pipe2.completed = False pipe2._increment_stage = mock.MagicMock(return_value=True) pipe2.current_stage = 1 pipe3 = mock.Mock() pipe3.lock = mt.Lock() pipe3.state = states.INITIAL pipe3.uid = 'pipe.0002' pipe3.completed = True pipe3._increment_stage = mock.MagicMock(return_value=True) pipe3.current_stage = 1 wfp._workflow = set([pipe2, pipe3]) stage = mock.Mock() stage.uid = 'stage.0000' stage.state = states.SCHEDULING stage.post_exec = mock.MagicMock( return_value=['pipe.0001', 'pipe.0002']) wfp._execute_post_exec(pipe, stage) exp_out = set([(pipe2, 'Pipeline', states.INITIAL), (pipe3, 'Pipeline', states.DONE)]) self.assertEqual(global_advs, exp_out)
def test_execute_workload(self, mocked_init, mocked_dumps, mocked_Logger, mocked_Reporter, mocked_BlockingConnection): global_advs = [] def _advance_side_effect(obj, obj_type, state): nonlocal global_advs global_advs.append([obj, obj_type, state]) wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._rmq_conn_params = 'test_rmq_params' wfp._pending_queue = ['test_queue'] wfp._logger = mocked_Logger wfp._advance = mock.MagicMock(side_effect=_advance_side_effect) stage = mock.Mock() stage.uid = 'stage.0000' stage.state = states.SCHEDULING task = mock.Mock() task.uid = 'task.0000' task.state = states.INITIAL workload = [task] stage.tasks = [task] scheduled_stages = [stage] wfp._execute_workload(workload, scheduled_stages) self.assertEqual(global_advs[0], [task, 'Task', 'SCHEDULED']) self.assertEqual(global_advs[1], [stage, 'Stage', 'SCHEDULED'])
def test_wfp_check_processor(): p = Pipeline() s = Stage() t = Task() t.executable = '/bin/date' s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, rmq_conn_params=amgr._rmq_conn_params, resubmit_failed=False) wfp.start_processor() assert wfp.check_processor() wfp.terminate_processor() assert not wfp.check_processor()
def test_wfp_enqueue(): p = Pipeline() s = Stage() t = Task() t.executable = ['/bin/date'] s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=False) wfp._initialize_workflow() amgr.workflow = [p] profiler = ru.Profiler(name='radical.entk.temp') for t in p.stages[0].tasks: assert t.state == states.INITIAL assert p.stages[0].state == states.INITIAL assert p.state == states.INITIAL amgr._terminate_sync = Event() sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread') sync_thread.start() proc = Process(target=func_for_enqueue_test, name='temp-proc', args=(wfp, )) proc.start() proc.join() amgr._terminate_sync.set() sync_thread.join() for t in p.stages[0].tasks: assert t.state == states.SCHEDULED assert p.stages[0].state == states.SCHEDULED assert p.state == states.SCHEDULING
def test_write_session_description(): amgr = AppManager(hostname=hostname, port=port) amgr.resource_desc = {'resource' : 'xsede.stampede', 'walltime' : 59, 'cpus' : 128, 'gpus' : 64, 'project' : 'xyz', 'queue' : 'high'} workflow = [generate_pipeline(1), generate_pipeline(2)] amgr.workflow = workflow amgr._wfp = WFprocessor(sid=amgr.sid, workflow=amgr._workflow, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, resubmit_failed=amgr._resubmit_failed, rmq_conn_params=amgr._rmq_conn_params) amgr._workflow = amgr._wfp.workflow amgr._task_manager = TaskManager(sid=amgr._sid, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, rmgr=amgr._rmgr, rmq_conn_params=amgr._rmq_conn_params ) write_session_description(amgr) desc = ru.read_json('%s/radical.entk.%s.json' % (amgr._sid, amgr._sid)) src = '%s/sample_data' % pwd assert desc == ru.read_json('%s/expected_desc_write_session.json' % src)
def test_wfp_dequeue(): p = Pipeline() s = Stage() t = Task() t.executable = '/bin/date' s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._hostname, port=amgr._port, resubmit_failed=False) wfp.initialize_workflow() assert p.state == states.INITIAL assert p.stages[0].state == states.INITIAL for t in p.stages[0].tasks: assert t.state == states.INITIAL p.state == states.SCHEDULED p.stages[0].state == states.SCHEDULING for t in p.stages[0].tasks: t.state = states.COMPLETED task_as_dict = json.dumps(t.to_dict()) mq_connection = pika.BlockingConnection( pika.ConnectionParameters(host=amgr._hostname, port=amgr._port)) mq_channel = mq_connection.channel() mq_channel.basic_publish(exchange='', routing_key='%s' % amgr._completed_queue[0], body=task_as_dict) wfp.start_processor() th = mt.Thread(target=func_for_dequeue_test, name='temp-proc', args=(p, )) th.start() th.join() wfp.terminate_processor() assert p.state == states.DONE assert p.stages[0].state == states.DONE for t in p.stages[0].tasks: assert t.state == states.DONE
def test_wfp_enqueue(): p = Pipeline() s = Stage() t = Task() t.executable = ['/bin/date'] s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=False) wfp._initialize_workflow() amgr.workflow = [p] profiler = ru.Profiler(name='radical.entk.temp') for t in p.stages[0].tasks: assert t.state == states.INITIAL assert p.stages[0].state == states.INITIAL assert p.state == states.INITIAL amgr._terminate_sync = Event() sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread') sync_thread.start() proc = Process(target=func_for_enqueue_test, name='temp-proc', args=(wfp,)) proc.start() proc.join() amgr._terminate_sync.set() sync_thread.join() for t in p.stages[0].tasks: assert t.state == states.SCHEDULED assert p.stages[0].state == states.SCHEDULED assert p.state == states.SCHEDULING
def test_wfp_initialization(self, mocked_generate_id, mocked_getcwd, mocked_Logger, mocked_Profiler, mocked_Reporter): wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) self.assertIsNone(wfp._wfp_process) self.assertIsNone(wfp._enqueue_thread) self.assertIsNone(wfp._dequeue_thread) self.assertIsNone(wfp._enqueue_thread_terminate) self.assertIsNone(wfp._dequeue_thread_terminate) self.assertEqual(wfp._rmq_ping_interval, 10) self.assertEqual(wfp._path, 'test_folder/test_sid') self.assertEqual(wfp._workflow, 'workflow') self.assertEqual(wfp._sid, 'test_sid') self.assertEqual(wfp._pending_queue, 'pending_queue') self.assertEqual(wfp._completed_queue, 'completed_queue') self.assertFalse(wfp._resubmit_failed) self.assertEqual(wfp._rmq_conn_params, 'test_rmq_params') self.assertEqual(wfp._uid, 'wfp.0000') os.environ['RMQ_PING_INTERVAL'] = '20' wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=True) self.assertIsNone(wfp._wfp_process) self.assertIsNone(wfp._enqueue_thread) self.assertIsNone(wfp._dequeue_thread) self.assertEqual(wfp._rmq_ping_interval, 20) self.assertEqual(wfp._path, 'test_folder/test_sid') self.assertEqual(wfp._workflow, 'workflow') self.assertEqual(wfp._sid, 'test_sid') self.assertEqual(wfp._pending_queue, 'pending_queue') self.assertEqual(wfp._completed_queue, 'completed_queue') self.assertTrue(wfp._resubmit_failed) self.assertEqual(wfp._rmq_conn_params, 'test_rmq_params') self.assertEqual(wfp._uid, 'wfp.0000')
def test_enqueue(self, mocked_init, mocked_Logger, mocked_Profiler, mocked_sleep, mocked_create_workload, mocked_execute_workload): wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._logger = mocked_Logger wfp._prof = mocked_Profiler wfp._uid = 'wfp.0000' wfp._enqueue_thread_terminate = mock.Mock() wfp._enqueue_thread_terminate.is_set = mock.MagicMock( side_effect=[False, True]) wfp._enqueue() with self.assertRaises(Exception): wfp._enqueue()
def test_wfp_initialize_workflow(): p = Pipeline() s = Stage() t = Task() t.executable = ['/bin/date'] s.add_tasks(t) p.add_stages(s) wfp = WFprocessor(sid='test', workflow=[p], pending_queue=list(), completed_queue=list(), mq_hostname=hostname, port=port, resubmit_failed=False) wfp._initialize_workflow() assert p.uid is not None assert p.stages[0].uid is not None for t in p.stages[0].tasks: assert t.uid is not None
def test_wfp_workflow_incomplete(): p = Pipeline() s = Stage() t = Task() t.executable = ['/bin/date'] s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=False) wfp._initialize_workflow() assert wfp.workflow_incomplete() amgr.workflow = [p] profiler = ru.Profiler(name='radical.entk.temp') p.stages[0].state == states.SCHEDULING p.state == states.SCHEDULED for t in p.stages[0].tasks: t.state = states.COMPLETED import json import pika task_as_dict = json.dumps(t.to_dict()) mq_connection = pika.BlockingConnection( pika.ConnectionParameters(host=amgr._mq_hostname, port=amgr._port)) mq_channel = mq_connection.channel() mq_channel.basic_publish(exchange='', routing_key='%s-completedq-1' % amgr._sid, body=task_as_dict) amgr._terminate_sync = Event() sync_thread = Thread(target=amgr._synchronizer, name='synchronizer-thread') sync_thread.start() proc = Process(target=func_for_dequeue_test, name='temp-proc', args=(wfp, )) proc.start() proc.join() amgr._terminate_sync.set() sync_thread.join() assert not wfp.workflow_incomplete()
def test_wfp_initialize_workflow(): p = Pipeline() s = Stage() t = Task() t.executable = '/bin/date' s.add_tasks(t) p.add_stages(s) rmq_conn_params = pika.ConnectionParameters(host=hostname, port=port) wfp = WFprocessor(sid='test', workflow=[p], pending_queue=list(), completed_queue=list(), rmq_conn_params=rmq_conn_params, resubmit_failed=False) wfp.initialize_workflow() assert p.uid is not None assert p.stages[0].uid is not None for t in p.stages[0].tasks: assert t.uid is not None
def test_write_workflow(): try: wf = list() wf.append(generate_pipeline(1)) wf.append(generate_pipeline(2)) amgr = AppManager(hostname=hostname, port=port) amgr.workflow = wf amgr._wfp = WFprocessor(sid=amgr._sid, workflow=amgr._workflow, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=amgr._resubmit_failed) amgr._wfp._initialize_workflow() wf = amgr._wfp.workflow write_workflow(wf, 'test') data = ru.read_json('test/entk_workflow.json') assert len(data) == len(wf) + 1 stack = data.pop(0) assert stack.keys() == ['stack'] assert stack['stack'].keys() == ['sys','radical'] assert stack['stack']['sys'].keys() == ["python","pythonpath","virtualenv"] assert stack['stack']['radical'].keys() == ['saga', 'radical.pilot', 'radical.utils', 'radical.entk'] p_cnt = 0 for p in data: assert p['uid'] == wf[p_cnt].uid assert p['name'] == wf[p_cnt].name assert p['state_history'] == wf[p_cnt].state_history s_cnt = 0 for s in p['stages']: assert s['uid'] == wf[p_cnt].stages[s_cnt].uid assert s['name'] == wf[p_cnt].stages[s_cnt].name assert s['state_history'] == wf[p_cnt].stages[s_cnt].state_history for t in wf[p_cnt].stages[s_cnt].tasks: assert t.to_dict() in s['tasks'] s_cnt += 1 p_cnt += 1 except Exception as ex: shutil.rmtree('test') raise
def test_wfp_enqueue(): p = Pipeline() s = Stage() t = Task() t.executable = '/bin/date' s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._hostname, port=amgr._port, resubmit_failed=False) wfp.initialize_workflow() assert p.state == states.INITIAL assert p.stages[0].state == states.INITIAL for t in p.stages[0].tasks: assert t.state == states.INITIAL wfp.start_processor() th = mt.Thread(target=func_for_enqueue_test, name='temp-proc', args=(p, )) th.start() th.join() wfp.terminate_processor() assert p.state == states.SCHEDULING assert p.stages[0].state == states.SCHEDULED for t in p.stages[0].tasks: assert t.state == states.SCHEDULED
def test_write_session_description(): amgr = AppManager(hostname=hostname, port=port, username=username, password=password) amgr.resource_desc = { 'resource': 'xsede.stampede', 'walltime': 59, 'cpus': 128, 'gpus': 64, 'project': 'xyz', 'queue': 'high' } workflow = [generate_pipeline(1), generate_pipeline(2)] amgr.workflow = workflow amgr._wfp = WFprocessor(sid=amgr.sid, workflow=amgr._workflow, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, resubmit_failed=amgr._resubmit_failed, rmq_conn_params=amgr._rmq_conn_params) amgr._workflow = amgr._wfp.workflow amgr._task_manager = TaskManager(sid=amgr._sid, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, rmgr=amgr._rmgr, rmq_conn_params=amgr._rmq_conn_params) write_session_description(amgr) desc = ru.read_json('%s/radical.entk.%s.json' % (amgr._sid, amgr._sid)) # tasks are originally set but saved as a list in json # uses sorting for convenient comparison, this doesn't change validity for k, v in (desc['tree'].items()): if k.startswith("stage"): desc['tree'][k]['children'] = sorted(v['children']) src = '%s/sample_data' % pwd assert desc == ru.read_json('%s/expected_desc_write_session.json' % src)
def test_write_session_description(): hostname = os.environ.get('RMQ_HOSTNAME', 'localhost') port = int(os.environ.get('RMQ_PORT', 5672)) amgr = AppManager(hostname=hostname, port=port) amgr.resource_desc = { 'resource': 'xsede.stampede', 'walltime': 60, 'cpus': 128, 'gpus': 64, 'project': 'xyz', 'queue': 'high' } workflow = [generate_pipeline(1), generate_pipeline(2)] amgr.workflow = workflow amgr._wfp = WFprocessor(sid=amgr._sid, workflow=amgr._workflow, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=amgr._resubmit_failed) amgr._wfp._initialize_workflow() amgr._workflow = amgr._wfp.workflow amgr._task_manager = TaskManager(sid=amgr._sid, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, rmgr=amgr._resource_manager, port=amgr._port ) # os.mkdir(amgr._sid) write_session_description(amgr) desc = ru.read_json('%s/radical.entk.%s.json' % (amgr._sid, amgr._sid)) curdir = os.path.dirname(os.path.abspath(__file__)) src = '%s/sample_data' % curdir assert desc == ru.read_json('%s/expected_desc_write_session.json' % src)
def test_wfp_workflow_incomplete(): p = Pipeline() s = Stage() t = Task() t.executable = '/bin/date' s.add_tasks(t) p.add_stages(s) amgr = Amgr(hostname=hostname, port=port, username=username, password=password) amgr._setup_mqs() wfp = WFprocessor(sid=amgr._sid, workflow=[p], pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, rmq_conn_params=amgr._rmq_conn_params, resubmit_failed=False) for t in p.stages[0].tasks: t.state = states.COMPLETED task_as_dict = json.dumps(t.to_dict()) credentials = pika.PlainCredentials(amgr._username, amgr._password) mq_connection = pika.BlockingConnection(pika.ConnectionParameters( host=amgr._hostname, port=amgr._port, credentials=credentials)) mq_channel = mq_connection.channel() mq_channel.basic_publish(exchange = '', routing_key = '%s' % amgr._completed_queue[0], body = task_as_dict) wfp.start_processor() th = mt.Thread(target=func_for_dequeue_test, name='temp-proc', args=(p,)) th.start() th.join() wfp.terminate_processor() assert not wfp.workflow_incomplete()
def test_write_workflow(): wf = list() wf.append(generate_pipeline(1)) wf.append(generate_pipeline(2)) amgr = AppManager(hostname=hostname, port=port) amgr.workflow = wf amgr._wfp = WFprocessor(sid=amgr._sid, workflow=amgr._workflow, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=amgr._resubmit_failed) amgr._wfp._initialize_workflow() wf = amgr._wfp.workflow write_workflow(wf, 'test') data = ru.read_json('test/entk_workflow.json') assert len(data) == len(wf) p_cnt = 0 for p in data: assert p['uid'] == wf[p_cnt].uid assert p['name'] == wf[p_cnt].name assert p['state_history'] == wf[p_cnt].state_history s_cnt = 0 for s in p['stages']: assert s['uid'] == wf[p_cnt].stages[s_cnt].uid assert s['name'] == wf[p_cnt].stages[s_cnt].name assert s['state_history'] == wf[p_cnt].stages[s_cnt].state_history for t in wf[p_cnt].stages[s_cnt].tasks: assert t.to_dict() in s['tasks'] s_cnt += 1 p_cnt += 1 shutil.rmtree('test')
def test_terminate_processor(self, mocked_init, mocked_Logger, mocked_Profiler): global_boolean = {} def _dequeue_side_effect(): nonlocal global_boolean time.sleep(0.1) global_boolean['dequeue'] = True def _enqueue_side_effect(): nonlocal global_boolean time.sleep(0.1) global_boolean['enqueue'] = True wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._uid = 'wfp.0000' wfp._logger = mocked_Logger wfp._prof = mocked_Profiler wfp._enqueue_thread = mt.Thread(target=_enqueue_side_effect) wfp._dequeue_thread = mt.Thread(target=_dequeue_side_effect) wfp._enqueue_thread_terminate = mt.Event() wfp._dequeue_thread_terminate = mt.Event() wfp._enqueue_thread.start() wfp._dequeue_thread.start() wfp.terminate_processor() self.assertTrue(wfp._enqueue_thread_terminate.is_set()) self.assertTrue(wfp._dequeue_thread_terminate.is_set()) self.assertIsNone(wfp._dequeue_thread) self.assertIsNone(wfp._enqueue_thread) self.assertTrue(global_boolean['dequeue']) self.assertTrue(global_boolean['enqueue'])
def test_start_processor(self, mocked_init, mocked_Logger, mocked_Profiler): global_boolean = {} def _dequeue_side_effect(): nonlocal global_boolean global_boolean['dequeue'] = True def _enqueue_side_effect(): nonlocal global_boolean global_boolean['enqueue'] = True wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._uid = 'wfp.0000' wfp._logger = mocked_Logger wfp._prof = mocked_Profiler wfp._enqueue_thread = None wfp._dequeue_thread = None wfp._enqueue_thread_terminate = None wfp._dequeue_thread_terminate = None wfp._dequeue = mock.MagicMock(side_effect=_dequeue_side_effect) wfp._enqueue = mock.MagicMock(side_effect=_enqueue_side_effect) wfp.start_processor() time.sleep(1) try: self.assertIsInstance(wfp._enqueue_thread_terminate, mt.Event) self.assertIsInstance(wfp._dequeue_thread_terminate, mt.Event) self.assertIsInstance(wfp._dequeue_thread, mt.Thread) self.assertIsInstance(wfp._enqueue_thread, mt.Thread) self.assertTrue(global_boolean['dequeue']) self.assertTrue(global_boolean['enqueue']) finally: if wfp._dequeue_thread.is_alive(): wfp._dequeue_thread.join() if wfp._enqueue_thread.is_alive(): wfp._enqueue_thread.join()
def test_update_dequeued_task(self, mocked_init, mocked_advance, mocked_Logger, mocked_Profiler): global_advs = list() def _advance_side_effect(obj, obj_type, state): nonlocal global_advs global_advs.append([obj, obj_type, state]) wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) wfp._uid = 'wfp.0000' wfp._logger = mocked_Logger wfp._prof = mocked_Profiler wfp._resubmit_failed = False wfp._advance = mock.MagicMock(side_effect=_advance_side_effect) pipe = mock.Mock() pipe.uid = 'pipe.0000' pipe.lock = mt.Lock() pipe.state = states.INITIAL pipe.completed = False pipe.current_stage = 1 pipe._increment_stage = mock.MagicMock(return_value=2) stage = mock.Mock() stage.uid = 'stage.0000' stage.state = states.SCHEDULING stage._check_stage_complete = mock.MagicMock(return_value=True) stage.post_exec = None task = mock.Mock() task.uid = 'task.0000' task.parent_pipeline = {'uid': 'pipe.0000'} task.parent_stage = {'uid': 'stage.0000'} task.state = states.INITIAL task.exit_code = 0 stage.tasks = [task] pipe.stages = [stage] wfp._workflow = set([pipe]) # Test for issue #271 wfp._update_dequeued_task(task) self.assertEqual(global_advs[0], [task, 'Task', states.DONE]) self.assertEqual(global_advs[1], [stage, 'Stage', states.DONE]) task.state = states.INITIAL task.exit_code = None wfp._update_dequeued_task(task) self.assertEqual(global_advs[2], [task, 'Task', states.INITIAL]) self.assertEqual(global_advs[3], [stage, 'Stage', states.DONE]) task.exit_code = 1 wfp._update_dequeued_task(task) self.assertEqual(global_advs[4], [task, 'Task', states.FAILED]) self.assertEqual(global_advs[5], [stage, 'Stage', states.DONE])
def test_write_session_description(): hostname = os.environ.get('RMQ_HOSTNAME', 'localhost') port = int(os.environ.get('RMQ_PORT', 5672)) amgr = AppManager(hostname=hostname, port=port) amgr.resource_desc = { 'resource': 'xsede.stampede', 'walltime': 60, 'cpus': 128, 'gpus': 64, 'project': 'xyz', 'queue': 'high' } workflow = [generate_pipeline(1), generate_pipeline(2)] amgr.workflow = workflow amgr._wfp = WFprocessor(sid=amgr._sid, workflow=amgr._workflow, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, port=amgr._port, resubmit_failed=amgr._resubmit_failed) amgr._wfp._initialize_workflow() amgr._workflow = amgr._wfp.workflow amgr._task_manager = TaskManager(sid=amgr._sid, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, mq_hostname=amgr._mq_hostname, rmgr=amgr._resource_manager, port=amgr._port) # os.mkdir(amgr._sid) write_session_description(amgr) desc = ru.read_json('%s/radical.entk.%s.json' % (amgr._sid, amgr._sid)) assert desc == { 'config': {}, 'entities': { 'appmanager': { 'event_model': {}, 'state_model': None, 'state_values': None }, 'pipeline': { 'event_model': {}, 'state_model': { 'CANCELED': 9, 'DESCRIBED': 1, 'DONE': 9, 'FAILED': 9, 'SCHEDULING': 2 }, 'state_values': { '1': 'DESCRIBED', '2': 'SCHEDULING', '9': ['DONE', 'CANCELED', 'FAILED'] } }, 'stage': { 'event_model': {}, 'state_model': { 'CANCELED': 9, 'DESCRIBED': 1, 'DONE': 9, 'FAILED': 9, 'SCHEDULED': 3, 'SCHEDULING': 2 }, 'state_values': { '1': 'DESCRIBED', '2': 'SCHEDULING', '3': 'SCHEDULED', '9': ['FAILED', 'CANCELED', 'DONE'] } }, 'task': { 'event_model': {}, 'state_model': { 'CANCELED': 9, 'DEQUEUED': 8, 'DEQUEUEING': 7, 'DESCRIBED': 1, 'DONE': 9, 'EXECUTED': 6, 'FAILED': 9, 'SCHEDULED': 3, 'SCHEDULING': 2, 'SUBMITTED': 5, 'SUBMITTING': 4 }, 'state_values': { '1': 'DESCRIBED', '2': 'SCHEDULING', '3': 'SCHEDULED', '4': 'SUBMITTING', '5': 'SUBMITTED', '6': 'EXECUTED', '7': 'DEQUEUEING', '8': 'DEQUEUED', '9': ['DONE', 'CANCELED', 'FAILED'] } } }, 'tree': { 'appmanager.0000': { 'cfg': {}, 'children': [ 'wfprocessor.0000', 'resource_manager.0000', 'task_manager.0000', 'pipeline.0000', 'pipeline.0001' ], 'etype': 'appmanager', 'has': [ 'pipeline', 'wfprocessor', 'resource_manager', 'task_manager' ], 'uid': 'appmanager.0000' }, 'pipeline.0000': { 'cfg': {}, 'children': ['stage.0000', 'stage.0001'], 'etype': 'pipeline', 'has': ['stage'], 'uid': 'pipeline.0000' }, 'pipeline.0001': { 'cfg': {}, 'children': ['stage.0002', 'stage.0003'], 'etype': 'pipeline', 'has': ['stage'], 'uid': 'pipeline.0001' }, 'resource_manager.0000': { 'cfg': {}, 'children': [], 'etype': 'resource_manager', 'has': [], 'uid': 'resource_manager.0000' }, 'stage.0000': { 'cfg': {}, 'children': ['task.0000'], 'etype': 'stage', 'has': ['task'], 'uid': 'stage.0000' }, 'stage.0001': { 'cfg': {}, 'children': [ 'task.0001', 'task.0002', 'task.0003', 'task.0004', 'task.0005', 'task.0006', 'task.0007', 'task.0008', 'task.0009', 'task.0010' ], 'etype': 'stage', 'has': ['task'], 'uid': 'stage.0001' }, 'stage.0002': { 'cfg': {}, 'children': ['task.0011'], 'etype': 'stage', 'has': ['task'], 'uid': 'stage.0002' }, 'stage.0003': { 'cfg': {}, 'children': [ 'task.0012', 'task.0013', 'task.0014', 'task.0015', 'task.0016', 'task.0017', 'task.0018', 'task.0019', 'task.0020', 'task.0021' ], 'etype': 'stage', 'has': ['task'], 'uid': 'stage.0003' }, 'task.0000': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0000' }, 'task.0001': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0001' }, 'task.0002': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0002' }, 'task.0003': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0003' }, 'task.0004': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0004' }, 'task.0005': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0005' }, 'task.0006': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0006' }, 'task.0007': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0007' }, 'task.0008': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0008' }, 'task.0009': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0009' }, 'task.0010': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0010' }, 'task.0011': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0011' }, 'task.0012': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0012' }, 'task.0013': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0013' }, 'task.0014': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0014' }, 'task.0015': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0015' }, 'task.0016': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0016' }, 'task.0017': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0017' }, 'task.0018': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0018' }, 'task.0019': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0019' }, 'task.0020': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0020' }, 'task.0021': { 'cfg': {}, 'children': [], 'etype': 'task', 'has': [], 'uid': 'task.0021' }, 'task_manager.0000': { 'cfg': {}, 'children': [], 'etype': 'task_manager', 'has': [], 'uid': 'task_manager.0000' }, 'wfprocessor.0000': { 'cfg': {}, 'children': [], 'etype': 'wfprocessor', 'has': [], 'uid': 'wfprocessor.0000' } } } shutil.rmtree(amgr._sid)
def test_write_workflows(): # -------------------------------------------------------------------------- def check_stack(stack): assert 'sys' in stack assert 'radical' in stack assert 'python' in stack['sys'] assert 'pythonpath' in stack['sys'] assert 'virtualenv' in stack['sys'] assert 'radical.utils' in stack['radical'] assert 'radical.saga' in stack['radical'] assert 'radical.pilot' in stack['radical'] assert 'radical.entk' in stack['radical'] # -------------------------------------------------------------------------- def check_wf(wf, check): for p_idx,p in enumerate(wf['pipes']): assert p['uid'] == check[p_idx].uid assert p['name'] == check[p_idx].name assert p['state_history'] == check[p_idx].state_history for s_idx,s in enumerate(p['stages']): assert s['uid'] == check[p_idx].stages[s_idx].uid assert s['name'] == check[p_idx].stages[s_idx].name assert s['state_history'] == check[p_idx].stages[s_idx].state_history for t in check[p_idx].stages[s_idx].tasks: assert t.to_dict() in s['tasks'] # -------------------------------------------------------------------------- try: wf = list() wf.append(generate_pipeline(1)) wf.append(generate_pipeline(2)) amgr = AppManager(hostname=hostname, port=port) amgr.workflow = wf amgr._wfp = WFprocessor(sid=amgr._sid, workflow=amgr._workflow, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, resubmit_failed=amgr._resubmit_failed, rmq_conn_params=amgr._rmq_conn_params) check = amgr.workflow # ---------------------------------------------------------------------- # check json output, with defaut and custom fname for fname in [None, 'wf.json']: write_workflows(amgr.workflows, 'test', fname=fname) if not fname: fname = 'entk_workflow.json' data = ru.read_json('test/%s' % fname) check_stack(data['stack']) check_wf(data['workflows'][0], check) assert len(data['workflows']) == 1 shutil.rmtree('test') # ---------------------------------------------------------------------- # check with data return data = write_workflows(amgr.workflows, 'test', fwrite=False) check_stack(data['stack']) check_wf(data['workflows'][0], check) assert len(data['workflows']) == 1 # ---------------------------------------------------------------------- # check with two workflows amgr.workflow = wf amgr._wfp = WFprocessor(sid=amgr._sid, workflow=amgr._workflow, pending_queue=amgr._pending_queue, completed_queue=amgr._completed_queue, resubmit_failed=amgr._resubmit_failed, rmq_conn_params=amgr._rmq_conn_params) check = amgr.workflows data = write_workflows(amgr.workflows, 'test', fwrite=False) check_stack(data['stack']) check_wf(data['workflows'][0], check[0]) check_wf(data['workflows'][1], check[0]) assert len(data['workflows']) == 2 shutil.rmtree('test') finally: try: shutil.rmtree('test') except: pass
def test_advance(self, mocked_init, mocked_Logger, mocked_Reporter): wfp = WFprocessor(sid='test_sid', workflow='workflow', pending_queue='pending_queue', completed_queue='completed_queue', rmq_conn_params='test_rmq_params', resubmit_failed=False) global_profs = [] def _log(log_entry, uid, state, msg): nonlocal global_profs global_profs.append([log_entry, uid, state, msg]) wfp._logger = mocked_Logger wfp._report = mocked_Reporter wfp._prof = mock.Mock() wfp._prof.prof = mock.MagicMock(side_effect=_log) wfp._uid = 'wfp.0000' obj = mock.Mock() obj.parent_stage = {'uid': 'test_stage'} obj.parent_pipeline = {'uid': 'test_pipe'} obj.uid = 'test_object' obj.state = 'test_state' wfp._advance(obj, 'Task', None) self.assertEqual(global_profs[0], ['advance', 'test_object', None, 'test_stage']) global_profs = [] wfp._advance(obj, 'Stage', 'new_state') self.assertEqual(global_profs[0], ['advance', 'test_object', 'new_state', 'test_pipe']) global_profs = [] wfp._advance(obj, 'Pipe', 'new_state') self.assertEqual(global_profs[0], ['advance', 'test_object', 'new_state', None])