def launch_worker(self, queue_name): config = DotDict() config.process.queue_name = queue_name config.process.buffer_limit = self.CFG.get_safe("service.ingestion_management.buffer_limit", 10) config.process.time_limit = self.CFG.get_safe("service.ingestion_management.time_limit", 10) process_definition_id, _ = self.clients.resource_registry.find_resources( restype=RT.ProcessDefinition, name="ingestion_worker_process", id_only=True ) validate_true(len(process_definition_id), "No process definition for ingestion workers could be found") process_definition_id = process_definition_id[0] process_id = self.clients.process_dispatcher.create_process(process_definition_id=process_definition_id) xn_ids, _ = self.clients.resource_registry.find_resources( restype=RT.ExchangeName, name=queue_name, id_only=True ) for xn_id in xn_ids: self.clients.resource_registry.create_association(xn_id, PRED.hasIngestionWorker, process_id) schedule = ProcessSchedule() schedule.restart_mode = ProcessRestartMode.ABNORMAL self.clients.process_dispatcher.schedule_process( process_definition_id=process_definition_id, schedule=schedule, process_id=process_id, configuration=config )
def _get_process_schedule(**kwargs): queueing_mode = kwargs.get('queueing_mode') restart_mode = kwargs.get('restart_mode') execution_engine_id = kwargs.get('execution_engine_id') node_exclusive = kwargs.get('node_exclusive') constraints = kwargs.get('constraints') process_schedule = ProcessSchedule() if queueing_mode is not None: try: process_schedule.queueing_mode = ProcessQueueingMode._value_map[queueing_mode] except KeyError: msg = "%s is not a known ProcessQueueingMode" % (queueing_mode) raise BadRequest(msg) if restart_mode is not None: try: process_schedule.restart_mode = ProcessRestartMode._value_map[restart_mode] except KeyError: msg = "%s is not a known ProcessRestartMode" % (restart_mode) raise BadRequest(msg) else: # if restart mode isn't specified, use NEVER. HA Agent itself will reschedule failures. process_schedule.restart_mode = ProcessRestartMode.NEVER target = ProcessTarget() if execution_engine_id is not None: target.execution_engine_id = execution_engine_id if node_exclusive is not None: target.node_exclusive = node_exclusive if constraints is not None: target.constraints = constraints process_schedule.target = target return process_schedule
def test_restart_mode(self): proc_def = DotDict() proc_def['name'] = "someprocess" proc_def['executable'] = {'module': 'my_module', 'class': 'class'} mock_read_definition = Mock() mock_read_definition.return_value = proc_def self.pd_service.backend.read_definition = mock_read_definition pid = self.pd_service.create_process("fake-process-def-id") pyon_restart_mode = ProcessRestartMode.ABNORMAL core_restart_mode = "ABNORMAL" proc_schedule = ProcessSchedule() proc_schedule.restart_mode = pyon_restart_mode configuration = {"some": "value"} pid2 = self.pd_service.schedule_process("fake-process-def-id", proc_schedule, configuration, pid) self.assertEqual(self.mock_core.schedule_process.call_count, 1) call_args, call_kwargs = self.mock_core.schedule_process.call_args self.assertEqual(call_kwargs['restart_mode'], core_restart_mode)
def test_node_exclusive(self): # the node_exclusive constraint is used to ensure multiple processes # of the same "kind" each get a VM exclusive of each other. Other # processes may run on these VMs, just not processes with the same # node_exclusive tag. Since we cannot directly query the contents # of each node in this test, we prove the capability by scheduling # processes one by one and checking their state. # verifies L4-CI-CEI-RQ121 # verifies L4-CI-CEI-RQ57 # first off, setUp() created a single node and eeagent. # We schedule two processes with the same "abc" node_exclusive # tag. Since there is only one node, the first process should run # and the second should be queued. process_target = ProcessTarget(execution_engine_id="engine1") process_target.node_exclusive = "abc" process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS process_schedule.target = process_target pid1 = self.pd_cli.create_process(self.process_definition_id) self.waiter.start() self.pd_cli.schedule_process(self.process_definition_id, process_schedule, process_id=pid1) self.waiter.await_state_event(pid1, ProcessStateEnum.RUNNING) pid2 = self.pd_cli.create_process(self.process_definition_id) self.pd_cli.schedule_process(self.process_definition_id, process_schedule, process_id=pid2) self.waiter.await_state_event(pid2, ProcessStateEnum.WAITING) # now demonstrate that the node itself is not full by launching # a third process without a node_exclusive tag -- it should start # immediately process_target.node_exclusive = None pid3 = self.pd_cli.create_process(self.process_definition_id) self.pd_cli.schedule_process(self.process_definition_id, process_schedule, process_id=pid3) self.waiter.await_state_event(pid3, ProcessStateEnum.RUNNING) # finally, add a second node to the engine. pid2 should be started # since there is an exclusive "abc" node free. node2_id = uuid.uuid4().hex self._send_node_state("engine1", node2_id) self._start_eeagent(node2_id) self.waiter.await_state_event(pid2, ProcessStateEnum.RUNNING) # kill the processes for good self.pd_cli.cancel_process(pid1) self.waiter.await_state_event(pid1, ProcessStateEnum.TERMINATED) self.pd_cli.cancel_process(pid2) self.waiter.await_state_event(pid2, ProcessStateEnum.TERMINATED) self.pd_cli.cancel_process(pid3) self.waiter.await_state_event(pid3, ProcessStateEnum.TERMINATED)
def _launch_process(self, queue_name='', out_streams=None, process_definition_id='', configuration=None): """ Launches the process """ # ------------------------------------------------------------------------------------ # Spawn Configuration and Parameters # ------------------------------------------------------------------------------------ if 'process' not in configuration: configuration['process'] = {} configuration['process']['queue_name'] = queue_name configuration['process']['publish_streams'] = out_streams # Setting the restart mode schedule = ProcessSchedule() schedule.restart_mode = ProcessRestartMode.ABNORMAL schedule.queueing_mode = ProcessQueueingMode.ALWAYS # ------------------------------------------------------------------------------------ # Process Spawning # ------------------------------------------------------------------------------------ # Spawn the process pid = self.clients.process_dispatcher.schedule_process( process_definition_id=process_definition_id, schedule=schedule, configuration=configuration) validate_is_not_none(pid, "Process could not be spawned") return pid
def _launch_process(self, queue_name='', out_streams=None, process_definition_id='', configuration=None): """ Launches the process """ # ------------------------------------------------------------------------------------ # Spawn Configuration and Parameters # ------------------------------------------------------------------------------------ if 'process' not in configuration: configuration['process'] = {} configuration['process']['queue_name'] = queue_name configuration['process']['publish_streams'] = out_streams # Setting the restart mode schedule = ProcessSchedule() schedule.restart_mode = ProcessRestartMode.ABNORMAL # ------------------------------------------------------------------------------------ # Process Spawning # ------------------------------------------------------------------------------------ # Spawn the process pid = self.clients.process_dispatcher.schedule_process( process_definition_id=process_definition_id, schedule= schedule, configuration=configuration ) validate_is_not_none( pid, "Process could not be spawned") return pid
def launch_worker(self, queue_name, config): config = DotDict(config or {}) config.process.queue_name = queue_name config.process.buffer_limit = self.CFG.get_safe( 'service.ingestion_management.buffer_limit', 10) config.process.time_limit = self.CFG.get_safe( 'service.ingestion_management.time_limit', 10) process_definition_id, _ = self.clients.resource_registry.find_resources( restype=RT.ProcessDefinition, name='ingestion_worker_process', id_only=True) validate_true( len(process_definition_id), 'No process definition for ingestion workers could be found') process_definition_id = process_definition_id[0] process_id = self.clients.process_dispatcher.create_process( process_definition_id=process_definition_id) xn_ids, _ = self.clients.resource_registry.find_resources( restype=RT.ExchangeName, name=queue_name, id_only=True) for xn_id in xn_ids: self.clients.resource_registry.create_association( xn_id, PRED.hasIngestionWorker, process_id) schedule = ProcessSchedule() schedule.restart_mode = ProcessRestartMode.ABNORMAL schedule.queueing_mode = ProcessQueueingMode.ALWAYS self.clients.process_dispatcher.schedule_process( process_definition_id=process_definition_id, schedule=schedule, process_id=process_id, configuration=config)
def test_create_schedule_cancel(self): process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS proc_name = 'myreallygoodname' pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid, name=proc_name) self.assertEqual(pid, pid2) # verifies L4-CI-CEI-RQ141 and L4-CI-CEI-RQ142 self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, {}) self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING) # make sure process is readable directly from RR (mirrored) # verifies L4-CI-CEI-RQ63 # verifies L4-CI-CEI-RQ64 proc = self.rr_cli.read(pid) self.assertEqual(proc.process_id, pid) # now try communicating with the process to make sure it is really running test_client = TestClient() for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # verifies L4-CI-CEI-RQ147 # check the process name was set in container got_proc_name = test_client.get_process_name(pid=pid2) self.assertEqual(proc_name, got_proc_name) # kill the process and start it again self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # kill the process for good self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED)
def test_create_schedule_cancel(self): process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) # verifies L4-CI-CEI-RQ141 and L4-CI-CEI-RQ142 self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, {}) self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING) # make sure process is readable directly from RR (mirrored) # verifies L4-CI-CEI-RQ63 # verifies L4-CI-CEI-RQ64 proc = self.rr_cli.read(pid) self.assertEqual(proc.process_id, pid) # now try communicating with the process to make sure it is really running test_client = TestClient() for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # verifies L4-CI-CEI-RQ147 # kill the process and start it again self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) self.waiter.stop() oldpid = pid pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.assertNotEqual(oldpid, pid) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # kill the process for good self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED)
def _add_test_process(self, restart_mode=None): process_schedule = ProcessSchedule() if restart_mode is not None: process_schedule.restart_mode = restart_mode pid = self.pd_cli.create_process(self.process_definition_id) pid_listen_name = "PDtestproc_%s" % uuid.uuid4().hex config = {'process': {'listen_name': pid_listen_name}} self.pd_cli.schedule_process(self.process_definition_id, process_schedule, process_id=pid, configuration=config) client = TestClient(to_name=pid_listen_name) return pid, client
def test_code_download(self): # create a process definition that has no URL; only module and class. process_definition_no_url = ProcessDefinition( name='test_process_nodownload') process_definition_no_url.executable = { 'module': 'ion.my.test.process', 'class': 'TestProcess' } process_definition_id_no_url = self.pd_cli.create_process_definition( process_definition_no_url) # create another that has a URL of the python file (this very file) # verifies L4-CI-CEI-RQ114 url = "file://%s" % os.path.join(os.path.dirname(__file__), 'test_process_dispatcher.py') process_definition = ProcessDefinition(name='test_process_download') process_definition.executable = { 'module': 'ion.my.test.process', 'class': 'TestProcess', 'url': url } process_definition_id = self.pd_cli.create_process_definition( process_definition) process_target = ProcessTarget() process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS process_schedule.target = process_target self.waiter.start() # Test a module with no download fails pid_no_url = self.pd_cli.create_process(process_definition_id_no_url) self.pd_cli.schedule_process(process_definition_id_no_url, process_schedule, process_id=pid_no_url) self.waiter.await_state_event(pid_no_url, ProcessStateEnum.FAILED) # Test a module with a URL runs pid = self.pd_cli.create_process(process_definition_id) self.pd_cli.schedule_process(process_definition_id, process_schedule, process_id=pid) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
def _launch_highcharts(self, viz_id, data_product_id, out_stream_id): ''' Launches the high-charts transform ''' stream_ids, _ = self.clients.resource_registry.find_objects( data_product_id, PRED.hasStream, id_only=True) if not stream_ids: raise BadRequest( "Can't launch high charts streaming: data product doesn't have associated stream (%s)" % data_product_id) queue_name = 'viz_%s' % data_product_id sub_id = self.clients.pubsub_management.create_subscription( name='viz transform for %s' % data_product_id, exchange_name=queue_name, stream_ids=stream_ids) self.clients.pubsub_management.activate_subscription(sub_id) self.clients.resource_registry.create_association( viz_id, PRED.hasSubscription, sub_id) config = DotDict() config.process.publish_streams.highcharts = out_stream_id config.process.queue_name = queue_name # This process MUST be launched the first time or fail so the user # doesn't wait there for nothing to happen. schedule = ProcessSchedule() schedule.restart_mode = ProcessRestartMode.NEVER schedule.queueing_mode = ProcessQueueingMode.NEVER # Launch the process procdef_id = self._get_highcharts_procdef() pid = self.clients.process_dispatcher.schedule_process( process_definition_id=procdef_id, schedule=schedule, configuration=config) # Make sure it launched or raise an error process_gate = ProcessStateGate( self.clients.process_dispatcher.read_process, pid, ProcessStateEnum.RUNNING) if not process_gate. await (self.CFG.get_safe( 'endpoint.receive.timeout', 10)): raise ServiceUnavailable( "Failed to launch high charts realtime visualization")
def test_node_exclusive_eeid(self): proc_def = DotDict() proc_def['name'] = "someprocess" proc_def['executable'] = {'module': 'my_module', 'class': 'class'} mock_read_definition = Mock() mock_read_definition.return_value = proc_def self.pd_service.backend.read_definition = mock_read_definition pid = self.pd_service.create_process("fake-process-def-id") node_exclusive = "someattr" ee_id = "some_ee" proc_schedule = ProcessSchedule() proc_schedule.target.node_exclusive = node_exclusive proc_schedule.target.execution_engine_id = ee_id configuration = {"some": "value"} pid2 = self.pd_service.schedule_process("fake-process-def-id", proc_schedule, configuration, pid) self.assertEqual(self.mock_core.schedule_process.call_count, 1) call_args, call_kwargs = self.mock_core.schedule_process.call_args self.assertEqual(call_kwargs['execution_engine_id'], ee_id) self.assertEqual(call_kwargs['node_exclusive'], node_exclusive)
def schedule_process(self, upid, definition_id, configuration=None, subscribers=None, constraints=None, queueing_mode=None, restart_mode=None, execution_engine_id=None, node_exclusive=None): definition = self.real_client.read_process_definition(definition_id) self.event_pub.publish_event(event_type="ProcessLifecycleEvent", origin=definition.name, origin_type="DispatchedHAProcess", state=ProcessStateEnum.RUNNING) create_upid = self.real_client.create_process(definition_id) process_schedule = ProcessSchedule() if queueing_mode is not None: try: process_schedule.queueing_mode = ProcessQueueingMode._value_map[queueing_mode] except KeyError: msg = "%s is not a known ProcessQueueingMode" % (queueing_mode) raise BadRequest(msg) if restart_mode is not None: try: process_schedule.restart_mode = ProcessRestartMode._value_map[restart_mode] except KeyError: msg = "%s is not a known ProcessRestartMode" % (restart_mode) raise BadRequest(msg) target = ProcessTarget() if execution_engine_id is not None: target.execution_engine_id = execution_engine_id if node_exclusive is not None: target.node_exclusive = node_exclusive if constraints is not None: target.constraints = constraints process_schedule.target = target sched_pid = self.real_client.schedule_process(definition_id, process_schedule, configuration=configuration, process_id=create_upid) proc = self.real_client.read_process(sched_pid) self._associate_process(proc) dict_proc = {'upid': proc.process_id, 'state': self.state_map.get(proc.process_state, self.unknown_state), } return dict_proc
def test_idempotency(self): # ensure every operation can be safely retried process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS proc_name = 'myreallygoodname' pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) # note: if we import UNSCHEDULED state into ProcessStateEnum, # this assertion will need to change. proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_state, ProcessStateEnum.REQUESTED) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid, name=proc_name) self.assertEqual(pid, pid2) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) # repeating schedule is harmless pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid, name=proc_name) self.assertEqual(pid, pid2) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, {}) self.assertEqual(proc.process_state, ProcessStateEnum.RUNNING) self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) # repeating cancel is harmless self.pd_cli.cancel_process(pid) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, {}) self.assertEqual(proc.process_state, ProcessStateEnum.TERMINATED)
def test_create_schedule_cancel(self): process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS pid = self.pd_cli.create_process(self.process_definition_id) self.subscribe_events(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.await_state_event(pid, ProcessStateEnum.SPAWN) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, {}) self.assertEqual(proc.process_state, ProcessStateEnum.SPAWN) # now try communicating with the process to make sure it is really running test_client = TestClient() for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # kill the process and start it again self.pd_cli.cancel_process(pid) self.await_state_event(pid, ProcessStateEnum.TERMINATE) oldpid = pid pid = self.pd_cli.create_process(self.process_definition_id) self.subscribe_events(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.assertNotEqual(oldpid, pid) self.await_state_event(pid, ProcessStateEnum.SPAWN) for i in range(5): self.assertEqual(i + 1, test_client.count(timeout=10)) # kill the process for good self.pd_cli.cancel_process(pid) self.await_state_event(pid, ProcessStateEnum.TERMINATE)
def test_schedule_bad_config(self): process_schedule = ProcessSchedule() # a non-JSON-serializable IonObject o = ProcessTarget() with self.assertRaises(BadRequest) as ar: self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={"bad": o}) self.assertTrue(ar.exception.message.startswith("bad configuration"))
def _launch_highcharts(self, viz_id, data_product_id, out_stream_id): ''' Launches the high-charts transform ''' stream_ids, _ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasStream, id_only=True) if not stream_ids: raise BadRequest("Can't launch high charts streaming: data product doesn't have associated stream (%s)" % data_product_id) queue_name = 'viz_%s' % data_product_id sub_id = self.clients.pubsub_management.create_subscription( name='viz transform for %s' % data_product_id, exchange_name=queue_name, stream_ids=stream_ids) self.clients.pubsub_management.activate_subscription(sub_id) self.clients.resource_registry.create_association(viz_id, PRED.hasSubscription, sub_id) config = DotDict() config.process.publish_streams.highcharts = out_stream_id config.process.queue_name = queue_name # This process MUST be launched the first time or fail so the user # doesn't wait there for nothing to happen. schedule = ProcessSchedule() schedule.restart_mode = ProcessRestartMode.NEVER schedule.queueing_mode = ProcessQueueingMode.NEVER # Launch the process procdef_id = self._get_highcharts_procdef() pid = self.clients.process_dispatcher.schedule_process( process_definition_id=procdef_id, schedule=schedule, configuration=config) # Make sure it launched or raise an error process_gate = ProcessStateGate(self.clients.process_dispatcher.read_process, pid, ProcessStateEnum.RUNNING) if not process_gate.await(self.CFG.get_safe('endpoint.receive.timeout', 10)): raise ServiceUnavailable("Failed to launch high charts realtime visualization")
def test_schedule_with_config(self): process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) # verifies L4-CI-CEI-RQ66 # feed in a string that the process will return -- verifies that # configuration actually makes it to the instantiated process test_response = uuid.uuid4().hex configuration = {"test_response": test_response} pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration=configuration, process_id=pid) self.assertEqual(pid, pid2) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) test_client = TestClient() # verifies L4-CI-CEI-RQ139 # assure that configuration block (which can contain inputs, outputs, # and arbitrary config) 1) makes it to the process and 2) is returned # in process queries self.assertEqual(test_client.query(), test_response) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, configuration) # kill the process for good self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED)
def test_schedule_cancel(self): process_schedule = ProcessSchedule() process_schedule.target = ProcessTarget() process_schedule.target.constraints = {'site' : 'chicago'} config = {'some': "value"} pid = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration=config) self.assertEqual(self.fake_pd.dispatch_process.call_count, 1) args, kwargs = self.fake_pd.dispatch_process.call_args self.assertFalse(args) self.assertEqual(set(kwargs), set(['upid', 'spec', 'subscribers', 'constraints'])) spec = kwargs['spec'] self.assertEqual(spec['run_type'], 'pyon_single') self.assertEqual(spec['parameters']['rel']['apps'][0]['config'], config) self.pd_cli.cancel_process(pid) self.fake_pd.terminate_process.assert_called_once_with(upid=pid)
def launch(self, agent_config, process_definition_id): """ schedule the launch """ log.debug("schedule agent process") process_schedule = ProcessSchedule(restart_mode=ProcessRestartMode.ABNORMAL, queueing_mode=ProcessQueueingMode.ALWAYS) process_id = self.process_dispatcher_client.schedule_process(process_definition_id=process_definition_id, schedule=process_schedule, configuration=agent_config) self.process_id = process_id return process_id
def test_schedule_with_config(self): process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start(pid) # verifies L4-CI-CEI-RQ66 # feed in a string that the process will return -- verifies that # configuration actually makes it to the instantiated process test_response = uuid.uuid4().hex configuration = {"test_response" : test_response} pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration=configuration, process_id=pid) self.assertEqual(pid, pid2) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) test_client = TestClient() # verifies L4-CI-CEI-RQ139 # assure that configuration block (which can contain inputs, outputs, # and arbitrary config) 1) makes it to the process and 2) is returned # in process queries self.assertEqual(test_client.query(), test_response) proc = self.pd_cli.read_process(pid) self.assertEqual(proc.process_id, pid) self.assertEqual(proc.process_configuration, configuration) # kill the process for good self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED)
def test_code_download(self): # create a process definition that has no URL; only module and class. process_definition_no_url = ProcessDefinition(name='test_process_nodownload') process_definition_no_url.executable = {'module': 'ion.my.test.process', 'class': 'TestProcess'} process_definition_id_no_url = self.pd_cli.create_process_definition(process_definition_no_url) # create another that has a URL of the python file (this very file) # verifies L4-CI-CEI-RQ114 url = "file://%s" % os.path.join(os.path.dirname(__file__), 'test_process_dispatcher.py') process_definition = ProcessDefinition(name='test_process_download') process_definition.executable = {'module': 'ion.my.test.process', 'class': 'TestProcess', 'url': url} process_definition_id = self.pd_cli.create_process_definition(process_definition) process_target = ProcessTarget() process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS process_schedule.target = process_target self.waiter.start() # Test a module with no download fails pid_no_url = self.pd_cli.create_process(process_definition_id_no_url) self.pd_cli.schedule_process(process_definition_id_no_url, process_schedule, process_id=pid_no_url) self.waiter.await_state_event(pid_no_url, ProcessStateEnum.FAILED) # Test a module with a URL runs pid = self.pd_cli.create_process(process_definition_id) self.pd_cli.schedule_process(process_definition_id, process_schedule, process_id=pid) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING)
def test_create_schedule_cancel(self): process_schedule = ProcessSchedule() pid = self.pd_cli.create_process(self.process_definition_id) self.subscribe_events(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.await_state_event(pid, ProcessStateEnum.SPAWN) # now try communicating with the process to make sure it is really running test_client = TestClient() for i in range(5): # this timeout may be too low self.assertEqual(i + 1, test_client.count(timeout=1)) # kill the process and start it again self.pd_cli.cancel_process(pid) self.await_state_event(pid, ProcessStateEnum.TERMINATE) oldpid = pid pid = self.pd_cli.create_process(self.process_definition_id) self.subscribe_events(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.assertNotEqual(oldpid, pid) self.await_state_event(pid, ProcessStateEnum.SPAWN) for i in range(5): # this timeout may be too low self.assertEqual(i + 1, test_client.count(timeout=1)) # kill the process for good self.pd_cli.cancel_process(pid) self.await_state_event(pid, ProcessStateEnum.TERMINATE)
def start_external_dataset_agent_instance(self, external_dataset_agent_instance_id=''): """Launch an dataset agent instance process and return its process id. Agent instance resource must exist and be associated with an external dataset @param external_dataset_agent_instance_id str @retval process_id str @throws NotFound object with specified id does not exist """ #todo: may want to call retrieve_external_dataset_agent_instance here #todo: if instance running, then return or throw #todo: if instance exists and dataset_agent_instance_obj.dataset_agent_config is completd then just schedule_process dataset_agent_instance_obj = self.clients.resource_registry.read(external_dataset_agent_instance_id) #retrieve the associated external dataset device ext_dataset_ids, _ = self.clients.resource_registry.find_subjects(RT.ExternalDataset, PRED.hasAgentInstance, external_dataset_agent_instance_id, True) if not ext_dataset_ids: raise NotFound("No External Dataset attached to this Dataset Agent Instance " + str(external_dataset_agent_instance_id)) if len(ext_dataset_ids) > 1: raise BadRequest("Dataset Agent Instance should only have ONE External Dataset" + str(external_dataset_agent_instance_id)) ext_dataset_id = ext_dataset_ids[0] log.debug("start_external_dataset_agent_instance: external dataset is %s connected to dataset agent instance %s ", str(ext_dataset_id), str(external_dataset_agent_instance_id)) #retrieve the external dataset model model_ids, _ = self.clients.resource_registry.find_objects(ext_dataset_id, PRED.hasModel, RT.ExternalDatasetModel, True) if not model_ids: raise NotFound("No External Dataset Model attached to this External Dataset " + str(ext_dataset_id)) ext_dataset_model_id = model_ids[0] log.debug("start_external_dataset_agent_instance:External Dataset Model %s" + str(ext_dataset_model_id)) #retrieve the associated instrument agent agent_ids, _ = self.clients.resource_registry.find_subjects(RT.ExternalDatasetAgent, PRED.hasModel, ext_dataset_model_id, True) if not agent_ids: raise NotFound("No External Dataset Agent attached to this External Dataset Model " + str(ext_dataset_model_id)) ext_dataset_agent_id = agent_ids[0] log.debug("start_external_dataset_agent_instance: external dataset agent '%s'" % ext_dataset_agent_id) #retrieve the associated process definition process_def_ids, _ = self.clients.resource_registry.find_objects(ext_dataset_agent_id, PRED.hasProcessDefinition, RT.ProcessDefinition, True) if not process_def_ids: raise NotFound("No Process Definition attached to this ExtDataset Agent " + str(ext_dataset_agent_id)) if len(process_def_ids) > 1: raise BadRequest("ExtDataset Agent should only have ONE Process Definition" + str(ext_dataset_agent_id)) process_definition_id = process_def_ids[0] log.debug("activate_instrument: agent process definition %s" + str(process_definition_id)) # retrieve the process definition information process_def_obj = self.clients.resource_registry.read(process_definition_id) out_streams = {} #retrieve the output products data_product_ids, _ = self.clients.resource_registry.find_objects(ext_dataset_id, PRED.hasOutputProduct, RT.DataProduct, True) if not data_product_ids: raise NotFound("No output Data Products attached to this External Dataset " + str(ext_dataset_id)) for product_id in data_product_ids: stream_ids, _ = self.clients.resource_registry.find_objects(product_id, PRED.hasStream, RT.Stream, True) log.debug("start_external_dataset_agent_instance:output stream ids: %s" + str(stream_ids)) #One stream per product ...for now. if not stream_ids: raise NotFound("No Stream attached to this Data Product " + str(product_id)) if len(stream_ids) > 1: raise BadRequest("Data Product should only have ONE Stream" + str(product_id)) # retrieve the stream stream_obj = self.clients.resource_registry.read(stream_ids[0]) out_streams['parsed'] = stream_ids[0] # Create agent config. dataset_agent_instance_obj.dataset_agent_config = { 'driver_config' : dataset_agent_instance_obj.dataset_driver_config, 'stream_config' : out_streams, 'agent' : {'resource_id': ext_dataset_id}, 'test_mode' : True } log.debug("start_external_dataset_agent_instance: agent_config %s ", str(dataset_agent_instance_obj.dataset_agent_config)) # Setting the restart mode schedule = ProcessSchedule() schedule.restart_mode = ProcessRestartMode.ABNORMAL pid = self.clients.process_dispatcher.schedule_process(process_definition_id=process_definition_id, schedule=schedule, configuration=dataset_agent_instance_obj.dataset_agent_config) log.debug("start_external_dataset_agent_instance: schedule_process %s", pid) # add the process id and update the resource dataset_agent_instance_obj.agent_process_id = pid self.update_external_dataset_agent_instance(dataset_agent_instance_obj) return pid
def test_requested_ee(self): # request non-default engine process_target = ProcessTarget(execution_engine_id="engine2") process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.ALWAYS process_schedule.target = process_target pid = self.pd_cli.create_process(self.process_definition_id) self.waiter.start() self.pd_cli.schedule_process(self.process_definition_id, process_schedule, process_id=pid) self.waiter.await_state_event(pid, ProcessStateEnum.WAITING) # request unknown engine, with NEVER queuing mode. The request # should be rejected. # verifies L4-CI-CEI-RQ52 process_target = ProcessTarget(execution_engine_id="not-a-real-ee") process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.NEVER process_schedule.target = process_target rejected_pid = self.pd_cli.create_process(self.process_definition_id) self.pd_cli.schedule_process(self.process_definition_id, process_schedule, process_id=rejected_pid) self.waiter.await_state_event(rejected_pid, ProcessStateEnum.REJECTED) # now add a node and eeagent for engine2. original process should leave # queue and start running node2_id = uuid.uuid4().hex self._send_node_state("engine2", node2_id) self._start_eeagent(node2_id) self.waiter.await_state_event(pid, ProcessStateEnum.RUNNING) # spawn another process. it should start immediately. process_target = ProcessTarget(execution_engine_id="engine2") process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.NEVER process_schedule.target = process_target pid2 = self.pd_cli.create_process(self.process_definition_id) self.pd_cli.schedule_process(self.process_definition_id, process_schedule, process_id=pid2) self.waiter.await_state_event(pid2, ProcessStateEnum.RUNNING) # one more with node exclusive process_target = ProcessTarget(execution_engine_id="engine2", node_exclusive="hats") process_schedule = ProcessSchedule() process_schedule.queueing_mode = ProcessQueueingMode.NEVER process_schedule.target = process_target pid3 = self.pd_cli.create_process(self.process_definition_id) self.pd_cli.schedule_process(self.process_definition_id, process_schedule, process_id=pid3) self.waiter.await_state_event(pid3, ProcessStateEnum.RUNNING) # kill the processes for good self.pd_cli.cancel_process(pid) self.waiter.await_state_event(pid, ProcessStateEnum.TERMINATED) self.pd_cli.cancel_process(pid2) self.waiter.await_state_event(pid2, ProcessStateEnum.TERMINATED) self.pd_cli.cancel_process(pid3) self.waiter.await_state_event(pid3, ProcessStateEnum.TERMINATED)