class ProcessDispatcherServiceBridgeTest(PyonTestCase):
"""Tests the bridge backend of the PD
"""
def setUp(self):
self.pd_service = ProcessDispatcherService()
self.pd_service.container = DotDict()
self.pd_service.container['resource_registry'] = Mock()
pdcfg = dict(uri="amqp://hello", topic="pd", exchange="123")
self.pd_service.CFG = DotDict()
self.pd_service.CFG['process_dispatcher_bridge'] = pdcfg
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDBridgeBackend)
self.pd_service.backend.rr = self.mock_rr = Mock()
self.event_pub = Mock()
self.pd_service.backend.event_pub = self.event_pub
# sneak in and replace dashi connection method
mock_dashi = Mock()
mock_dashi.consume.return_value = lambda: None
self.pd_service.backend._init_dashi = lambda: mock_dashi
self.mock_dashi = mock_dashi
self.pd_service.start()
self.assertEqual(mock_dashi.handle.call_count, 1)
def tearDown(self):
self.pd_service.quit()
def test_create_schedule(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
mock_read_def = Mock()
mock_read_def.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_def
pid = self.pd_service.create_process("fake-process-def-id")
mock_read_def.assert_called_once_with("fake-process-def-id")
proc_schedule = DotDict()
proc_schedule['target'] = DotDict()
proc_schedule.target['constraints'] = {"hats": 4}
proc_schedule.target['node_exclusive'] = None
proc_schedule.target['execution_engine_id'] = None
configuration = {"some": "value"}
name = 'allthehats'
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid, name=name)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(pid, pid2)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(self.mock_dashi.call.call_count, 1)
call_args, call_kwargs = self.mock_dashi.call.call_args
self.assertEqual(set(call_kwargs),
set(['upid', 'definition_id', 'configuration', 'subscribers', 'constraints', 'name']))
self.assertEqual(call_kwargs['constraints'],
proc_schedule.target['constraints'])
self.assertEqual(call_kwargs['subscribers'],
self.pd_service.backend.pd_process_subscribers)
self.assertEqual(call_args, ("pd", "schedule_process"))
self.assertEqual(self.event_pub.publish_event.call_count, 0)
# trigger some fake async state updates from dashi
self.pd_service.backend._process_state(dict(upid=pid,
state="400-PENDING"))
self.assertEqual(self.event_pub.publish_event.call_count, 1)
self.pd_service.backend._process_state(dict(upid=pid,
state="500-RUNNING"))
self.assertEqual(self.event_pub.publish_event.call_count, 2)
def test_cancel(self):
ok = self.pd_service.cancel_process("process-id")
self.assertTrue(ok)
self.mock_dashi.call.assert_called_once_with("pd", "terminate_process",
upid="process-id")
self.assertEqual(self.event_pub.publish_event.call_count, 0)
self.pd_service.backend._process_state(dict(upid="process-id",
state="700-TERMINATED"))
self.assertEqual(self.event_pub.publish_event.call_count, 1)
def test_definitions(self):
executable = {'module': 'my_module', 'class': 'class'}
definition = ProcessDefinition(name="someprocess", executable=executable)
pd_id = self.pd_service.create_process_definition(definition)
assert self.mock_dashi.call.called
self.assertTrue(pd_id)
assert self.mock_rr.create.called_once_with(definition, object_id=pd_id)
self.mock_dashi.call.reset_mock()
self.mock_dashi.call.return_value = dict(name="someprocess",
executable=executable)
definition2 = self.pd_service.read_process_definition("someprocess")
assert self.mock_dashi.call.called
self.assertEqual(definition2.name, "someprocess")
self.assertEqual(definition2.executable, executable)
self.mock_dashi.call.reset_mock()
self.pd_service.delete_process_definition("someprocess")
assert self.mock_dashi.call.called
assert self.mock_rr.delete.called_once_with(pd_id)
def test_read_process(self):
self.mock_dashi.call.return_value = dict(upid="processid",
state="500-RUNNING")
proc = self.pd_service.read_process("processid")
assert self.mock_dashi.call.called
self.assertEqual(proc.process_id, "processid")
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
self.assertEqual(proc.process_configuration, {})
def test_read_process_notfound(self):
self.mock_dashi.call.return_value = None
with self.assertRaises(NotFound):
self.pd_service.read_process("processid")
assert self.mock_dashi.call.called
def test_read_process_with_config(self):
config = {"hats": 4}
self.mock_dashi.call.return_value = dict(upid="processid",
state="500-RUNNING", configuration=config)
proc = self.pd_service.read_process("processid")
assert self.mock_dashi.call.called
self.assertEqual(proc.process_id, "processid")
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
self.assertEqual(proc.process_configuration, config)
def test_list_processes(self):
core_procs = [dict(upid="processid", state="500-RUNNING")]
self.mock_dashi.call.return_value = core_procs
procs = self.pd_service.list_processes()
proc = procs[0]
self.assertEqual(proc.process_id, "processid")
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
class ProcessDispatcherServiceTest(PyonTestCase):
def setUp(self):
mock_clients = self._create_service_mock('process_dispatcher')
self.pd_service = ProcessDispatcherService()
self.pd_service.clients = mock_clients
self.pd_service.container = DotDict()
self.pd_service.container['spawn_process'] = Mock()
self.pd_service.container['id'] = 'mock_container_id'
self.pd_service.container['proc_manager'] = DotDict()
self.pd_service.container.proc_manager['terminate_process'] = Mock()
self.pd_service.container.proc_manager['procs'] = {}
# CRUD Shortcuts
self.mock_rr_create = self.pd_service.clients.resource_registry.create
self.mock_rr_read = self.pd_service.clients.resource_registry.read
self.mock_rr_update = self.pd_service.clients.resource_registry.update
self.mock_rr_delete = self.pd_service.clients.resource_registry.delete
self.mock_rr_find = self.pd_service.clients.resource_registry.find_objects
self.mock_rr_find_res = self.pd_service.clients.resource_registry.find_resources
self.mock_rr_assoc = self.pd_service.clients.resource_registry.find_associations
self.mock_rr_create_assoc = self.pd_service.clients.resource_registry.create_association
self.mock_rr_del_assoc = self.pd_service.clients.resource_registry.delete_association
self.mock_cc_spawn = self.pd_service.container.spawn_process
self.mock_cc_terminate = self.pd_service.container.proc_manager.terminate_process
self.mock_cc_procs = self.pd_service.container.proc_manager.procs
def test_local_create_schedule(self):
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDLocalBackend)
event_pub = Mock()
self.pd_service.backend.event_pub = event_pub
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module':'my_module', 'class':'class'}
self.mock_rr_read.return_value = proc_def
self.mock_cc_spawn.return_value = '123'
pid = self.pd_service.create_process("fake-process-def-id")
# not used for anything in local mode
proc_schedule = DotDict()
configuration = {"some": "value"}
self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(self.mock_cc_spawn.call_count, 1)
call_args, call_kwargs = self.mock_cc_spawn.call_args
self.assertFalse(call_args)
# name should be def name followed by a uuid
name = call_kwargs['name']
self.assertEqual(name, pid)
self.assertEqual(len(call_kwargs), 5)
self.assertEqual(call_kwargs['module'], 'my_module')
self.assertEqual(call_kwargs['cls'], 'class')
called_config = call_kwargs['config']
self.assertEqual(called_config, configuration)
self.assertEqual(event_pub.publish_event.call_count, 1)
def test_schedule_process_notfound(self):
proc_schedule = DotDict()
configuration = {}
self.mock_rr_read.side_effect = NotFound()
with self.assertRaises(NotFound):
self.pd_service.schedule_process("not-a-real-process-id",
proc_schedule, configuration)
self.mock_rr_read.assert_called_once_with("not-a-real-process-id", "")
def test_local_cancel(self):
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDLocalBackend)
ok = self.pd_service.cancel_process("process-id")
self.assertTrue(ok)
self.mock_cc_terminate.assert_called_once_with("process-id")
def test_bridge_create_schedule(self):
pdcfg = dict(uri="amqp://hello", topic="pd", exchange="123")
self.pd_service.CFG = DotDict()
self.pd_service.CFG['process_dispatcher_bridge'] = pdcfg
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDBridgeBackend)
event_pub = Mock()
self.pd_service.backend.event_pub = event_pub
# sneak in and replace dashi connection method
mock_dashi = Mock()
mock_dashi.consume.return_value = lambda : None
self.pd_service.backend._init_dashi = lambda : mock_dashi
self.pd_service.start()
self.assertEqual(mock_dashi.handle.call_count, 1)
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module':'my_module', 'class':'class'}
self.mock_rr_read.return_value = proc_def
pid = self.pd_service.create_process("fake-process-def-id")
proc_schedule = DotDict()
proc_schedule['target'] = DotDict()
proc_schedule.target['constraints'] = {"hats" : 4}
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(pid, pid2)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(mock_dashi.call.call_count, 1)
call_args, call_kwargs = mock_dashi.call.call_args
self.assertEqual(set(call_kwargs),
set(['upid', 'spec', 'subscribers', 'constraints']))
self.assertEqual(call_kwargs['constraints'],
proc_schedule.target['constraints'])
self.assertEqual(call_kwargs['subscribers'],
self.pd_service.backend.pd_process_subscribers)
self.assertEqual(call_args, ("pd", "dispatch_process"))
self.assertEqual(event_pub.publish_event.call_count, 0)
# trigger some fake async state updates from dashi. first
# should not trigger an event
self.pd_service.backend._process_state(dict(upid=pid,
state="400-PENDING"))
self.assertEqual(event_pub.publish_event.call_count, 0)
self.pd_service.backend._process_state(dict(upid=pid,
state="500-RUNNING"))
self.assertEqual(event_pub.publish_event.call_count, 1)
def test_bridge_cancel(self):
pdcfg = dict(uri="amqp://hello", topic="pd", exchange="123")
self.pd_service.CFG = DotDict()
self.pd_service.CFG['process_dispatcher_bridge'] = pdcfg
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDBridgeBackend)
event_pub = Mock()
self.pd_service.backend.event_pub = event_pub
# sneak in and replace dashi connection method
mock_dashi = Mock()
mock_dashi.consume.return_value = lambda : None
self.pd_service.backend._init_dashi = lambda : mock_dashi
self.pd_service.start()
ok = self.pd_service.cancel_process("process-id")
self.assertTrue(ok)
mock_dashi.call.assert_called_once_with("pd", "terminate_process",
upid="process-id")
self.assertEqual(event_pub.publish_event.call_count, 0)
self.pd_service.backend._process_state(dict(upid="process-id",
state="700-TERMINATED"))
self.assertEqual(event_pub.publish_event.call_count, 1)
class ProcessDispatcherServiceLocalTest(PyonTestCase):
"""Tests the local backend of the PD
"""
def setUp(self):
self.pd_service = ProcessDispatcherService()
self.pd_service.container = DotDict()
self.pd_service.container['spawn_process'] = Mock()
self.pd_service.container['id'] = 'mock_container_id'
self.pd_service.container['proc_manager'] = DotDict()
self.pd_service.container['resource_registry'] = Mock()
self.pd_service.container.proc_manager['terminate_process'] = Mock()
self.pd_service.container.proc_manager['procs'] = {}
self.mock_cc_spawn = self.pd_service.container.spawn_process
self.mock_cc_terminate = self.pd_service.container.proc_manager.terminate_process
self.mock_cc_procs = self.pd_service.container.proc_manager.procs
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDLocalBackend)
self.pd_service.backend.rr = self.mock_rr = Mock()
def test_create_schedule(self):
backend = self.pd_service.backend
assert isinstance(backend, PDLocalBackend)
event_pub = Mock()
backend.event_pub = event_pub
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
self.mock_rr.read.return_value = proc_def
self.mock_cc_spawn.return_value = '123'
pid = self.pd_service.create_process("fake-process-def-id")
# not used for anything in local mode
proc_schedule = DotDict()
configuration = {"some": "value"}
if backend.SPAWN_DELAY:
with patch("gevent.spawn_later") as mock_gevent:
self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertTrue(mock_gevent.called)
self.assertEqual(mock_gevent.call_args[0][0], backend.SPAWN_DELAY)
self.assertEqual(mock_gevent.call_args[0][1], backend._inner_spawn)
spawn_args = mock_gevent.call_args[0][2:]
# now call the delayed spawn directly
backend._inner_spawn(*spawn_args)
else:
self.pd_service.schedule_process("fake-process-def-id", proc_schedule,
configuration, pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(self.mock_cc_spawn.call_count, 1)
call_args, call_kwargs = self.mock_cc_spawn.call_args
self.assertFalse(call_args)
# name should be def name followed by a uuid
name = call_kwargs['name']
self.assertEqual(name, proc_def['name'])
self.assertEqual(len(call_kwargs), 5)
self.assertEqual(call_kwargs['module'], 'my_module')
self.assertEqual(call_kwargs['cls'], 'class')
self.assertEqual(call_kwargs['process_id'], pid)
called_config = call_kwargs['config']
self.assertEqual(called_config, configuration)
# PENDING followed by RUNNING
self.assertEqual(event_pub.publish_event.call_count, 2)
process = self.pd_service.read_process(pid)
self.assertEqual(process.process_id, pid)
self.assertEqual(process.process_state, ProcessStateEnum.RUNNING)
def test_read_process_notfound(self):
with self.assertRaises(NotFound):
self.pd_service.read_process("processid")
def test_schedule_process_notfound(self):
proc_schedule = DotDict()
configuration = {}
self.mock_rr.read.side_effect = NotFound()
with self.assertRaises(NotFound):
self.pd_service.schedule_process("not-a-real-process-id",
proc_schedule, configuration)
self.mock_rr.read.assert_called_once_with("not-a-real-process-id")
def test_local_cancel(self):
ok = self.pd_service.cancel_process("process-id")
self.assertTrue(ok)
self.mock_cc_terminate.assert_called_once_with("process-id")
class ProcessDispatcherServiceNativeTest(PyonTestCase):
"""Tests the Pyon backend of the PD
"""
def setUp(self):
self.pd_service = ProcessDispatcherService()
self.pd_service.container = DotDict()
self.pd_service.container['spawn_process'] = Mock()
self.pd_service.container['id'] = 'mock_container_id'
self.pd_service.container['proc_manager'] = DotDict()
self.pd_service.container['resource_registry'] = Mock()
self.pd_service.container.proc_manager['terminate_process'] = Mock()
self.pd_service.container.proc_manager['procs'] = {}
pdcfg = dict(dashi_uri="amqp://hello", dashi_exchange="123",
static_resources=True, backend="native")
self.pd_service.CFG = DotDict()
self.pd_service.CFG['processdispatcher'] = pdcfg
self.mock_dashi = Mock()
with patch.multiple('ion.services.cei.process_dispatcher_service',
get_dashi=DEFAULT, ProcessDispatcherCore=DEFAULT,
ProcessDispatcherStore=DEFAULT, EngineRegistry=DEFAULT,
PDMatchmaker=DEFAULT) as mocks:
mocks['get_dashi'].return_value = self.mock_dashi
mocks['ProcessDispatcherStore'].return_value = self.mock_store = Mock()
mocks['ProcessDispatcherCore'].return_value = self.mock_core = Mock()
mocks['PDMatchmaker'].return_value = self.mock_matchmaker = Mock()
mocks['EngineRegistry'].return_value = self.mock_engineregistry = Mock()
self.pd_service.init()
# replace the core and matchmaker with mocks
self.pd_service.backend.beat_subscriber = self.mock_beat_subscriber = Mock()
self.assertIsInstance(self.pd_service.backend, PDNativeBackend)
self.pd_service.backend.rr = self.mock_rr = Mock()
self.event_pub = Mock()
self.pd_service.backend.event_pub = self.event_pub
self.pd_service.start()
self.assertEqual(self.mock_dashi.handle.call_count, 1)
self.mock_matchmaker.start_election.assert_called_once_with()
self.mock_beat_subscriber.start.assert_called_once_with()
def test_create_schedule(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class', 'url': 'myurl'}
mock_read_definition = Mock()
mock_read_definition.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_definition
pid = self.pd_service.create_process("fake-process-def-id")
proc_schedule = DotDict()
proc_schedule['target'] = DotDict()
proc_schedule.target['constraints'] = {"hats": 4}
proc_schedule.target['node_exclusive'] = None
proc_schedule.target['execution_engine_id'] = None
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(pid, pid2)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
def test_queueing_mode(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
mock_read_definition = Mock()
mock_read_definition.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_definition
pid = self.pd_service.create_process("fake-process-def-id")
pyon_queueing_mode = ProcessQueueingMode.ALWAYS
core_queueing_mode = "ALWAYS"
proc_schedule = ProcessSchedule()
proc_schedule.queueing_mode = pyon_queueing_mode
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
call_args, call_kwargs = self.mock_core.schedule_process.call_args
self.assertEqual(call_kwargs['queueing_mode'], core_queueing_mode)
def test_restart_mode(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
mock_read_definition = Mock()
mock_read_definition.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_definition
pid = self.pd_service.create_process("fake-process-def-id")
pyon_restart_mode = ProcessRestartMode.ABNORMAL
core_restart_mode = "ABNORMAL"
proc_schedule = ProcessSchedule()
proc_schedule.restart_mode = pyon_restart_mode
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
call_args, call_kwargs = self.mock_core.schedule_process.call_args
self.assertEqual(call_kwargs['restart_mode'], core_restart_mode)
def test_node_exclusive_eeid(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
mock_read_definition = Mock()
mock_read_definition.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_definition
pid = self.pd_service.create_process("fake-process-def-id")
node_exclusive = "someattr"
ee_id = "some_ee"
proc_schedule = ProcessSchedule()
proc_schedule.target.node_exclusive = node_exclusive
proc_schedule.target.execution_engine_id = ee_id
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
call_args, call_kwargs = self.mock_core.schedule_process.call_args
self.assertEqual(call_kwargs['execution_engine_id'], ee_id)
self.assertEqual(call_kwargs['node_exclusive'], node_exclusive)
def test_cancel(self):
ok = self.pd_service.cancel_process("process-id")
self.assertTrue(ok)
self.assertEqual(self.mock_core.terminate_process.call_count, 1)
def test_definitions(self):
executable = {'module': 'my_module', 'class': 'class'}
definition = ProcessDefinition(name="someprocess", executable=executable)
pd_id = self.pd_service.create_process_definition(definition)
assert self.mock_core.create_definition.called
self.assertTrue(pd_id)
assert self.mock_rr.create.called_once_with(definition, object_id=pd_id)
self.mock_core.describe_definition.return_value = dict(name="someprocess",
executable=executable)
definition2 = self.pd_service.read_process_definition("someprocess")
assert self.mock_core.describe_definition.called
self.assertEqual(definition2.name, "someprocess")
self.assertEqual(definition2.executable, executable)
self.pd_service.delete_process_definition("someprocess")
assert self.mock_core.remove_definition.called
assert self.mock_rr.delete.called_once_with(pd_id)
def test_read_process(self):
self.mock_core.describe_process.return_value = dict(upid="processid",
state="500-RUNNING")
proc = self.pd_service.read_process("processid")
assert self.mock_core.describe_process.called
self.assertEqual(proc.process_id, "processid")
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
self.assertEqual(proc.process_configuration, {})
def test_read_process_notfound(self):
self.mock_core.describe_process.return_value = None
with self.assertRaises(NotFound):
proc = self.pd_service.read_process("processid")
assert self.mock_core.describe_process.called
def test_read_process_with_config(self):
config = {"hats": 4}
self.mock_core.describe_process.return_value = dict(upid="processid",
state="500-RUNNING", configuration=config)
proc = self.pd_service.read_process("processid")
assert self.mock_core.describe_process.called
self.assertEqual(proc.process_id, "processid")
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
self.assertEqual(proc.process_configuration, config)
def test_list_processes(self):
core_procs = [dict(upid="processid", state="500-RUNNING")]
self.mock_core.describe_processes.return_value = core_procs
procs = self.pd_service.list_processes()
proc = procs[0]
self.assertEqual(proc.process_id, "processid")
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
class ProcessDispatcherServiceTest(PyonTestCase):
def setUp(self):
mock_clients = self._create_service_mock('process_dispatcher')
self.pd_service = ProcessDispatcherService()
self.pd_service.clients = mock_clients
self.pd_service.container = DotDict()
self.pd_service.container['spawn_process'] = Mock()
self.pd_service.container['id'] = 'mock_container_id'
self.pd_service.container['proc_manager'] = DotDict()
self.pd_service.container.proc_manager['terminate_process'] = Mock()
self.pd_service.container.proc_manager['procs'] = {}
# CRUD Shortcuts
self.mock_rr_create = self.pd_service.clients.resource_registry.create
self.mock_rr_read = self.pd_service.clients.resource_registry.read
self.mock_rr_update = self.pd_service.clients.resource_registry.update
self.mock_rr_delete = self.pd_service.clients.resource_registry.delete
self.mock_rr_find = self.pd_service.clients.resource_registry.find_objects
self.mock_rr_find_res = self.pd_service.clients.resource_registry.find_resources
self.mock_rr_assoc = self.pd_service.clients.resource_registry.find_associations
self.mock_rr_create_assoc = self.pd_service.clients.resource_registry.create_association
self.mock_rr_del_assoc = self.pd_service.clients.resource_registry.delete_association
self.mock_cc_spawn = self.pd_service.container.spawn_process
self.mock_cc_terminate = self.pd_service.container.proc_manager.terminate_process
self.mock_cc_procs = self.pd_service.container.proc_manager.procs
def test_local_create_schedule(self):
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDLocalBackend)
event_pub = Mock()
self.pd_service.backend.event_pub = event_pub
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
self.mock_rr_read.return_value = proc_def
self.mock_cc_spawn.return_value = '123'
pid = self.pd_service.create_process("fake-process-def-id")
# not used for anything in local mode
proc_schedule = DotDict()
configuration = {"some": "value"}
self.pd_service.schedule_process("fake-process-def-id", proc_schedule,
configuration, pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(self.mock_cc_spawn.call_count, 1)
call_args, call_kwargs = self.mock_cc_spawn.call_args
self.assertFalse(call_args)
# name should be def name followed by a uuid
name = call_kwargs['name']
self.assertEqual(name, pid)
self.assertEqual(len(call_kwargs), 5)
self.assertEqual(call_kwargs['module'], 'my_module')
self.assertEqual(call_kwargs['cls'], 'class')
called_config = call_kwargs['config']
self.assertEqual(called_config, configuration)
self.assertEqual(event_pub.publish_event.call_count, 1)
def test_schedule_process_notfound(self):
proc_schedule = DotDict()
configuration = {}
self.mock_rr_read.side_effect = NotFound()
with self.assertRaises(NotFound):
self.pd_service.schedule_process("not-a-real-process-id",
proc_schedule, configuration)
self.mock_rr_read.assert_called_once_with("not-a-real-process-id", "")
def test_local_cancel(self):
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDLocalBackend)
ok = self.pd_service.cancel_process("process-id")
self.assertTrue(ok)
self.mock_cc_terminate.assert_called_once_with("process-id")
def test_bridge_create_schedule(self):
pdcfg = dict(uri="amqp://hello", topic="pd", exchange="123")
self.pd_service.CFG = DotDict()
self.pd_service.CFG['process_dispatcher_bridge'] = pdcfg
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDBridgeBackend)
event_pub = Mock()
self.pd_service.backend.event_pub = event_pub
# sneak in and replace dashi connection method
mock_dashi = Mock()
mock_dashi.consume.return_value = lambda: None
self.pd_service.backend._init_dashi = lambda: mock_dashi
self.pd_service.start()
self.assertEqual(mock_dashi.handle.call_count, 1)
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
self.mock_rr_read.return_value = proc_def
pid = self.pd_service.create_process("fake-process-def-id")
proc_schedule = DotDict()
proc_schedule['target'] = DotDict()
proc_schedule.target['constraints'] = {"hats": 4}
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration,
pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(pid, pid2)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(mock_dashi.call.call_count, 1)
call_args, call_kwargs = mock_dashi.call.call_args
self.assertEqual(set(call_kwargs),
set(['upid', 'spec', 'subscribers', 'constraints']))
self.assertEqual(call_kwargs['constraints'],
proc_schedule.target['constraints'])
self.assertEqual(call_kwargs['subscribers'],
self.pd_service.backend.pd_process_subscribers)
self.assertEqual(call_args, ("pd", "dispatch_process"))
self.assertEqual(event_pub.publish_event.call_count, 0)
# trigger some fake async state updates from dashi. first
# should not trigger an event
self.pd_service.backend._process_state(
dict(upid=pid, state="400-PENDING"))
self.assertEqual(event_pub.publish_event.call_count, 0)
self.pd_service.backend._process_state(
dict(upid=pid, state="500-RUNNING"))
self.assertEqual(event_pub.publish_event.call_count, 1)
def test_bridge_cancel(self):
pdcfg = dict(uri="amqp://hello", topic="pd", exchange="123")
self.pd_service.CFG = DotDict()
self.pd_service.CFG['process_dispatcher_bridge'] = pdcfg
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDBridgeBackend)
event_pub = Mock()
self.pd_service.backend.event_pub = event_pub
# sneak in and replace dashi connection method
mock_dashi = Mock()
mock_dashi.consume.return_value = lambda: None
self.pd_service.backend._init_dashi = lambda: mock_dashi
self.pd_service.start()
ok = self.pd_service.cancel_process("process-id")
self.assertTrue(ok)
mock_dashi.call.assert_called_once_with("pd",
"terminate_process",
upid="process-id")
self.assertEqual(event_pub.publish_event.call_count, 0)
self.pd_service.backend._process_state(
dict(upid="process-id", state="700-TERMINATED"))
self.assertEqual(event_pub.publish_event.call_count, 1)
class ProcessDispatcherServiceLocalTest(PyonTestCase):
"""Tests the local backend of the PD
"""
def setUp(self):
self.pd_service = ProcessDispatcherService()
self.pd_service.container = DotDict()
self.pd_service.container['spawn_process'] = Mock()
self.pd_service.container['id'] = 'mock_container_id'
self.pd_service.container['proc_manager'] = DotDict()
self.pd_service.container['resource_registry'] = Mock()
self.pd_service.container.proc_manager['terminate_process'] = Mock()
self.pd_service.container.proc_manager['procs'] = {}
self.mock_cc_spawn = self.pd_service.container.spawn_process
self.mock_cc_terminate = self.pd_service.container.proc_manager.terminate_process
self.mock_cc_procs = self.pd_service.container.proc_manager.procs
self.pd_service.init()
self.assertIsInstance(self.pd_service.backend, PDLocalBackend)
self.pd_service.backend.rr = self.mock_rr = Mock()
self.pd_service.backend.event_pub = self.mock_event_pub = Mock()
def test_create_schedule(self):
backend = self.pd_service.backend
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
self.mock_rr.read.return_value = proc_def
self.mock_cc_spawn.return_value = '123'
pid = self.pd_service.create_process("fake-process-def-id")
# not used for anything in local mode
proc_schedule = DotDict()
configuration = {"some": "value"}
if backend.SPAWN_DELAY:
with patch("gevent.spawn_later") as mock_gevent:
self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration,
pid)
self.assertTrue(mock_gevent.called)
self.assertEqual(mock_gevent.call_args[0][0],
backend.SPAWN_DELAY)
self.assertEqual(mock_gevent.call_args[0][1],
backend._inner_spawn)
spawn_args = mock_gevent.call_args[0][2:]
# now call the delayed spawn directly
backend._inner_spawn(*spawn_args)
else:
self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration, pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(self.mock_cc_spawn.call_count, 1)
call_args, call_kwargs = self.mock_cc_spawn.call_args
self.assertFalse(call_args)
# name should be def name followed by a uuid
name = call_kwargs['name']
assert name.startswith(proc_def['name'])
self.assertEqual(len(call_kwargs), 5)
self.assertEqual(call_kwargs['module'], 'my_module')
self.assertEqual(call_kwargs['cls'], 'class')
self.assertEqual(call_kwargs['process_id'], pid)
called_config = call_kwargs['config']
self.assertEqual(called_config, configuration)
# PENDING followed by RUNNING
self.assertEqual(self.mock_event_pub.publish_event.call_count, 2)
process = self.pd_service.read_process(pid)
self.assertEqual(process.process_id, pid)
self.assertEqual(process.process_state, ProcessStateEnum.RUNNING)
def test_read_process_notfound(self):
with self.assertRaises(NotFound):
self.pd_service.read_process("processid")
def test_schedule_process_notfound(self):
proc_schedule = DotDict()
configuration = {}
self.mock_rr.read.side_effect = NotFound()
with self.assertRaises(NotFound):
self.pd_service.schedule_process("not-a-real-process-id",
proc_schedule, configuration)
self.mock_rr.read.assert_called_once_with("not-a-real-process-id")
def test_local_cancel(self):
pid = self.pd_service.create_process("fake-process-def-id")
ok = self.pd_service.cancel_process(pid)
self.assertTrue(ok)
self.mock_cc_terminate.assert_called_once_with(pid)
class ProcessDispatcherServiceNativeTest(PyonTestCase):
"""Tests the Pyon backend of the PD
"""
def setUp(self):
self.pd_service = ProcessDispatcherService()
self.pd_service.container = DotDict()
self.pd_service.container['spawn_process'] = Mock()
self.pd_service.container['id'] = 'mock_container_id'
self.pd_service.container['proc_manager'] = DotDict()
self.pd_service.container['resource_registry'] = Mock()
self.pd_service.container.proc_manager['terminate_process'] = Mock()
self.pd_service.container.proc_manager['procs'] = {}
pdcfg = dict(dashi_uri="amqp://hello",
dashi_exchange="123",
static_resources=True,
backend="native")
self.pd_service.CFG = DotDict()
self.pd_service.CFG['processdispatcher'] = pdcfg
self.mock_dashi = Mock()
with patch.multiple('ion.services.cei.process_dispatcher_service',
get_dashi=DEFAULT,
ProcessDispatcherCore=DEFAULT,
get_processdispatcher_store=DEFAULT,
EngineRegistry=DEFAULT,
PDMatchmaker=DEFAULT,
PDDoctor=DEFAULT) as mocks:
mocks['get_dashi'].return_value = self.mock_dashi
mocks[
'get_processdispatcher_store'].return_value = self.mock_store = Mock(
)
mocks[
'ProcessDispatcherCore'].return_value = self.mock_core = Mock(
)
mocks['PDMatchmaker'].return_value = self.mock_matchmaker = Mock()
mocks['PDDoctor'].return_value = self.mock_doctor = Mock()
mocks[
'EngineRegistry'].return_value = self.mock_engineregistry = Mock(
)
self.pd_service.init()
# replace the core and matchmaker with mocks
self.pd_service.backend.beat_subscriber = self.mock_beat_subscriber = Mock(
)
self.assertIsInstance(self.pd_service.backend, PDNativeBackend)
self.pd_service.backend.rr = self.mock_rr = Mock()
self.event_pub = Mock()
self.pd_service.backend.event_pub = self.event_pub
self.pd_service.start()
self.assertEqual(self.mock_dashi.handle.call_count, 1)
self.mock_matchmaker.start_election.assert_called_once_with()
self.mock_beat_subscriber.start.assert_called_once_with()
def test_create_schedule(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {
'module': 'my_module',
'class': 'class',
'url': 'myurl'
}
mock_read_definition = Mock()
mock_read_definition.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_definition
pid = self.pd_service.create_process("fake-process-def-id")
proc_schedule = DotDict()
proc_schedule['target'] = DotDict()
proc_schedule.target['constraints'] = {"hats": 4}
proc_schedule.target['node_exclusive'] = None
proc_schedule.target['execution_engine_id'] = None
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration,
pid)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(pid, pid2)
self.assertTrue(pid.startswith(proc_def.name) and pid != proc_def.name)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
def test_schedule_haagent_name(self):
haa_proc_def = DotDict()
haa_proc_def['name'] = "haagent"
haa_proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
payload_proc_def = DotDict()
payload_proc_def['name'] = "payload_process"
payload_proc_def['executable'] = {
'module': 'my_module',
'class': 'class'
}
proc_defs = {
"haa_proc_def_id": haa_proc_def,
"payload_proc_def_id": payload_proc_def
}
read_definition_mock = Mock()
read_definition_mock.side_effect = proc_defs.get
self.pd_service.backend.read_definition = read_definition_mock
# not used for anything in local mode
proc_schedule = DotDict()
configuration = {
"highavailability": {
"process_definition_id": "payload_proc_def_id"
}
}
self.pd_service.schedule_process("haa_proc_def_id", proc_schedule,
configuration)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
name = self.mock_core.schedule_process.call_args[1]['name']
self.assertTrue(name.startswith("payload_process-ha"))
# now try with scheduling by process definition name instead of ID
self.mock_core.schedule_process.reset_mock()
configuration = {
"highavailability": {
"process_definition_name": "payload_process"
}
}
self.pd_service.schedule_process("haa_proc_def_id", proc_schedule,
configuration)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
name = self.mock_core.schedule_process.call_args[1]['name']
self.assertTrue(name.startswith("payload_process-ha"))
def test_queueing_mode(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
mock_read_definition = Mock()
mock_read_definition.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_definition
pid = self.pd_service.create_process("fake-process-def-id")
pyon_queueing_mode = ProcessQueueingMode.ALWAYS
core_queueing_mode = "ALWAYS"
proc_schedule = ProcessSchedule()
proc_schedule.queueing_mode = pyon_queueing_mode
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration,
pid)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
call_args, call_kwargs = self.mock_core.schedule_process.call_args
self.assertEqual(call_kwargs['queueing_mode'], core_queueing_mode)
def test_restart_mode(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
mock_read_definition = Mock()
mock_read_definition.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_definition
pid = self.pd_service.create_process("fake-process-def-id")
pyon_restart_mode = ProcessRestartMode.ABNORMAL
core_restart_mode = "ABNORMAL"
proc_schedule = ProcessSchedule()
proc_schedule.restart_mode = pyon_restart_mode
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration,
pid)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
call_args, call_kwargs = self.mock_core.schedule_process.call_args
self.assertEqual(call_kwargs['restart_mode'], core_restart_mode)
def test_node_exclusive_eeid(self):
proc_def = DotDict()
proc_def['name'] = "someprocess"
proc_def['executable'] = {'module': 'my_module', 'class': 'class'}
mock_read_definition = Mock()
mock_read_definition.return_value = proc_def
self.pd_service.backend.read_definition = mock_read_definition
pid = self.pd_service.create_process("fake-process-def-id")
node_exclusive = "someattr"
ee_id = "some_ee"
proc_schedule = ProcessSchedule()
proc_schedule.target.node_exclusive = node_exclusive
proc_schedule.target.execution_engine_id = ee_id
configuration = {"some": "value"}
pid2 = self.pd_service.schedule_process("fake-process-def-id",
proc_schedule, configuration,
pid)
self.assertEqual(self.mock_core.schedule_process.call_count, 1)
call_args, call_kwargs = self.mock_core.schedule_process.call_args
self.assertEqual(call_kwargs['execution_engine_id'], ee_id)
self.assertEqual(call_kwargs['node_exclusive'], node_exclusive)
def test_cancel(self):
ok = self.pd_service.cancel_process("process-id")
self.assertTrue(ok)
self.assertEqual(self.mock_core.terminate_process.call_count, 1)
def test_definitions(self):
executable = {'module': 'my_module', 'class': 'class'}
definition = ProcessDefinition(name="someprocess",
executable=executable)
pd_id = self.pd_service.create_process_definition(definition)
assert self.mock_core.create_definition.called
self.assertTrue(pd_id)
assert self.mock_rr.create.called_once_with(definition,
object_id=pd_id)
self.mock_core.describe_definition.return_value = dict(
name="someprocess", executable=executable)
definition2 = self.pd_service.read_process_definition("someprocess")
assert self.mock_core.describe_definition.called
self.assertEqual(definition2.name, "someprocess")
self.assertEqual(definition2.executable, executable)
self.pd_service.delete_process_definition("someprocess")
assert self.mock_core.remove_definition.called
assert self.mock_rr.delete.called_once_with(pd_id)
def test_read_process(self):
self.mock_core.describe_process.return_value = dict(
upid="processid", state="500-RUNNING")
proc = self.pd_service.read_process("processid")
assert self.mock_core.describe_process.called
self.assertEqual(proc.process_id, "processid")
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
self.assertEqual(proc.process_configuration, {})
def test_read_process_notfound(self):
self.mock_core.describe_process.return_value = None
with self.assertRaises(NotFound):
proc = self.pd_service.read_process("processid")
assert self.mock_core.describe_process.called
def test_read_process_with_config(self):
config = {"hats": 4}
self.mock_core.describe_process.return_value = dict(
upid="processid", state="500-RUNNING", configuration=config)
proc = self.pd_service.read_process("processid")
assert self.mock_core.describe_process.called
self.assertEqual(proc.process_id, "processid")
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)
self.assertEqual(proc.process_configuration, config)
def test_list_processes(self):
core_procs = [dict(upid="processid", state="500-RUNNING")]
self.mock_core.describe_processes.return_value = core_procs
procs = self.pd_service.list_processes()
proc = procs[0]
self.assertEqual(proc.process_id, "processid")
self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING)