def run_reverse_transform(self): ''' Runs a reverse transform example and displays the results of performing the transform ''' tms_cli = TransformManagementServiceClient(node=self.container.node) procd_cli = ProcessDispatcherServiceClient(node=self.container.node) #------------------------------- # Process Definition #------------------------------- process_definition = IonObject(RT.ProcessDefinition, name='transform_process_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class': 'ReverseTransform' } process_definition_id = procd_cli.create_process_definition( process_definition) #------------------------------- # Execute Transform #------------------------------- input = [1, 2, 3, 4] retval = tms_cli.execute_transform( process_definition_id=process_definition_id, data=[1, 2, 3, 4], configuration={}) log.debug('Transform Input: %s', input) log.debug('Transform Output: %s', retval)
def run_reverse_transform(self): ''' Runs a reverse transform example and displays the results of performing the transform ''' tms_cli = TransformManagementServiceClient(node=self.container.node) procd_cli = ProcessDispatcherServiceClient(node=self.container.node) #------------------------------- # Process Definition #------------------------------- process_definition = IonObject(RT.ProcessDefinition, name='transform_process_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'ReverseTransform' } process_definition_id = procd_cli.create_process_definition(process_definition) #------------------------------- # Execute Transform #------------------------------- input = [1,2,3,4] retval = tms_cli.execute_transform(process_definition_id=process_definition_id, data=[1,2,3,4], configuration={}) log.debug('Transform Input: %s', input) log.debug('Transform Output: %s', retval)
class TransformManagementServiceIntTest(IonIntegrationTestCase): def setUp(self): # set up the container self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) self.tms_cli = TransformManagementServiceClient(node=self.container.node) self.rr_cli = ResourceRegistryServiceClient(node=self.container.node) self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node) self.input_stream_id = self.pubsub_cli.create_stream(name='input_stream',original=True) self.input_subscription_id = self.pubsub_cli.create_subscription(query=StreamQuery(stream_ids=[self.input_stream_id]),exchange_name='transform_input',name='input_subscription') self.output_stream_id = self.pubsub_cli.create_stream(name='output_stream',original=True) self.process_definition = ProcessDefinition(name='basic_transform_definition') self.process_definition.executable = {'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample'} self.process_definition_id = self.procd_cli.create_process_definition(process_definition=self.process_definition) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_create_transform(self): configuration = {'program_args':{'arg1':'value'}} transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id) # test transform creation in rr transform = self.rr_cli.read(transform_id) self.assertEquals(transform.name,'test_transform') # test associations predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition] assocs = [] for p in predicates: assocs += self.rr_cli.find_associations(transform_id,p,id_only=True) self.assertEquals(len(assocs),3) # test process creation transform = self.tms_cli.read_transform(transform_id) pid = transform.process_id proc = self.container.proc_manager.procs.get(pid) self.assertIsInstance(proc,TransformExample) # clean up self.tms_cli.delete_transform(transform_id) def test_create_transform_no_procdef(self): with self.assertRaises(NotFound): self.tms_cli.create_transform(name='test',in_subscription_id=self.input_subscription_id) def test_create_transform_bad_procdef(self): with self.assertRaises(NotFound): self.tms_cli.create_transform(name='test', in_subscription_id=self.input_subscription_id, process_definition_id='bad') def test_create_transform_no_config(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id, ) self.tms_cli.delete_transform(transform_id) def test_create_transform_name_failure(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id, ) with self.assertRaises(BadRequest): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id, ) self.tms_cli.delete_transform(transform_id) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_create_no_output(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, process_definition_id=self.process_definition_id, ) predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition] assocs = [] for p in predicates: assocs += self.rr_cli.find_associations(transform_id,p,id_only=True) self.assertEquals(len(assocs),2) # test process creation transform = self.tms_cli.read_transform(transform_id) pid = transform.process_id proc = self.container.proc_manager.procs.get(pid) self.assertIsInstance(proc,TransformExample) self.tms_cli.delete_transform(transform_id) def test_read_transform_exists(self): trans_obj = IonObject(RT.Transform,name='trans_obj') trans_id, _ = self.rr_cli.create(trans_obj) res = self.tms_cli.read_transform(trans_id) actual = self.rr_cli.read(trans_id) self.assertEquals(res._id,actual._id) def test_read_transform_nonexist(self): with self.assertRaises(NotFound) as e: self.tms_cli.read_transform('123') def test_activate_transform(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id ) self.tms_cli.activate_transform(transform_id) # pubsub check if activated? self.tms_cli.delete_transform(transform_id) def test_activate_transform_nonexist(self): with self.assertRaises(NotFound): self.tms_cli.activate_transform('1234') def test_delete_transform(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, process_definition_id=self.process_definition_id ) self.tms_cli.delete_transform(transform_id) # assertions with self.assertRaises(NotFound): self.rr_cli.read(transform_id) def test_delete_transform_nonexist(self): with self.assertRaises(NotFound): self.tms_cli.delete_transform('123') def test_execute_transform(self): # set up process_definition = ProcessDefinition(name='procdef_execute') process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example' process_definition.executable['class'] = 'ReverseTransform' data = [1,2,3] process_definition_id, _ = self.rr_cli.create(process_definition) retval = self.tms_cli.execute_transform(process_definition_id,data) self.assertEquals(retval,[3,2,1]) def test_integrated_transform(self): ''' This example script runs a chained three way transform: B A < C Where A is the even_odd transform (generates a stream of even and odd numbers from input) and B and C are the basic transforms that receive even and odd input ''' cc = self.container assertions = self.assertTrue pubsub_cli = PubsubManagementServiceClient(node=cc.node) rr_cli = ResourceRegistryServiceClient(node=cc.node) tms_cli = TransformManagementServiceClient(node=cc.node) #------------------------------- # Process Definition #------------------------------- # Create the process definition for the basic transform process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample' } basic_transform_definition_id, _ = rr_cli.create(process_definition) # Create The process definition for the TransformEvenOdd process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformEvenOdd' } evenodd_transform_definition_id, _ = rr_cli.create(process_definition) #------------------------------- # Streams #------------------------------- streams = [pubsub_cli.create_stream() for i in xrange(5)] #------------------------------- # Subscriptions #------------------------------- query = StreamQuery(stream_ids=[streams[0]]) input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue') query = StreamQuery(stream_ids = [streams[1]]) # even output even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue') query = StreamQuery(stream_ids = [streams[2]]) # odd output odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue') #------------------------------- # Launch the EvenOdd Transform #------------------------------- evenodd_id = tms_cli.create_transform(name='even_odd', in_subscription_id=input_subscription_id, out_streams={'even':streams[1], 'odd':streams[2]}, process_definition_id=evenodd_transform_definition_id, configuration={}) tms_cli.activate_transform(evenodd_id) #------------------------------- # Launch the Even Processing Transform #------------------------------- even_transform_id = tms_cli.create_transform(name='even_transform', in_subscription_id = even_subscription_id, out_streams={'even_plus1':streams[3]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(even_transform_id) #------------------------------- # Launch the Odd Processing Transform #------------------------------- odd_transform_id = tms_cli.create_transform(name='odd_transform', in_subscription_id = odd_subscription_id, out_streams={'odd_plus1':streams[4]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(odd_transform_id) #------------------------------- # Set up final subscribers #------------------------------- evenplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[3]]), exchange_name='evenplus1_queue', name='EvenPlus1Subscription', description='EvenPlus1 SubscriptionDescription' ) oddplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[4]]), exchange_name='oddplus1_queue', name='OddPlus1Subscription', description='OddPlus1 SubscriptionDescription' ) total_msg_count = 2 msgs = gevent.queue.Queue() def even1_message_received(message, headers): input = int(message.get('num')) assertions( (input % 2) ) # Assert it is odd (transform adds 1) msgs.put(True) def odd1_message_received(message, headers): input = int(message.get('num')) assertions(not (input % 2)) # Assert it is even msgs.put(True) subscriber_registrar = StreamSubscriberRegistrar(process=cc, container=cc) even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received) odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received) # Start subscribers even_subscriber.start() odd_subscriber.start() # Activate subscriptions pubsub_cli.activate_subscription(evenplus1_subscription_id) pubsub_cli.activate_subscription(oddplus1_subscription_id) #------------------------------- # Set up fake stream producer #------------------------------- pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, container=cc) stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0]) #------------------------------- # Start test #------------------------------- # Publish a stream for i in xrange(total_msg_count): stream_publisher.publish({'num':str(i)}) time.sleep(0.5) for i in xrange(total_msg_count * 2): try: msgs.get() except Empty: assertions(False, "Failed to process all messages correctly.") """
class TransformManagementServiceIntTest(IonIntegrationTestCase): def setUp(self): # set up the container self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) self.tms_cli = TransformManagementServiceClient(node=self.container.node) self.rr_cli = ResourceRegistryServiceClient(node=self.container.node) self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node) self.input_stream_id = self.pubsub_cli.create_stream(name='input_stream',original=True) self.input_subscription_id = self.pubsub_cli.create_subscription(query=StreamQuery(stream_ids=[self.input_stream_id]),exchange_name='transform_input',name='input_subscription') self.output_stream_id = self.pubsub_cli.create_stream(name='output_stream',original=True) self.process_definition = ProcessDefinition(name='basic_transform_definition') self.process_definition.executable = {'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample'} self.process_definition_id = self.procd_cli.create_process_definition(process_definition=self.process_definition) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_create_transform(self): configuration = {'program_args':{'arg1':'value'}} transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id) # test transform creation in rr transform = self.rr_cli.read(transform_id) self.assertEquals(transform.name,'test_transform') # test associations predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition] assocs = [] for p in predicates: assocs += self.rr_cli.find_associations(transform_id,p,id_only=True) self.assertEquals(len(assocs),3) # test process creation transform = self.tms_cli.read_transform(transform_id) pid = transform.process_id proc = self.container.proc_manager.procs.get(pid) self.assertIsInstance(proc,TransformExample) # clean up self.tms_cli.delete_transform(transform_id) def test_create_transform_no_procdef(self): with self.assertRaises(NotFound): self.tms_cli.create_transform(name='test',in_subscription_id=self.input_subscription_id) def test_create_transform_bad_procdef(self): with self.assertRaises(NotFound): self.tms_cli.create_transform(name='test', in_subscription_id=self.input_subscription_id, process_definition_id='bad') def test_create_transform_no_config(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id, ) self.tms_cli.delete_transform(transform_id) def test_create_transform_name_failure(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id, ) with self.assertRaises(BadRequest): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id, ) self.tms_cli.delete_transform(transform_id) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_create_no_output(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, process_definition_id=self.process_definition_id, ) predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition] assocs = [] for p in predicates: assocs += self.rr_cli.find_associations(transform_id,p,id_only=True) self.assertEquals(len(assocs),2) # test process creation transform = self.tms_cli.read_transform(transform_id) pid = transform.process_id proc = self.container.proc_manager.procs.get(pid) self.assertIsInstance(proc,TransformExample) self.tms_cli.delete_transform(transform_id) def test_read_transform_exists(self): trans_obj = IonObject(RT.Transform,name='trans_obj') trans_id, _ = self.rr_cli.create(trans_obj) res = self.tms_cli.read_transform(trans_id) actual = self.rr_cli.read(trans_id) self.assertEquals(res._id,actual._id) def test_read_transform_nonexist(self): with self.assertRaises(NotFound) as e: self.tms_cli.read_transform('123') def test_activate_transform(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, out_streams={'output':self.output_stream_id}, process_definition_id=self.process_definition_id ) self.tms_cli.activate_transform(transform_id) # pubsub check if activated? self.tms_cli.delete_transform(transform_id) def test_activate_transform_nonexist(self): with self.assertRaises(NotFound): self.tms_cli.activate_transform('1234') def test_delete_transform(self): transform_id = self.tms_cli.create_transform( name='test_transform', in_subscription_id=self.input_subscription_id, process_definition_id=self.process_definition_id ) self.tms_cli.delete_transform(transform_id) # assertions with self.assertRaises(NotFound): self.rr_cli.read(transform_id) def test_delete_transform_nonexist(self): with self.assertRaises(NotFound): self.tms_cli.delete_transform('123') def test_execute_transform(self): # set up process_definition = ProcessDefinition(name='procdef_execute') process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example' process_definition.executable['class'] = 'ReverseTransform' data = [1,2,3] process_definition_id, _ = self.rr_cli.create(process_definition) retval = self.tms_cli.execute_transform(process_definition_id,data) self.assertEquals(retval,[3,2,1]) def test_integrated_transform(self): ''' This example script runs a chained three way transform: B A < C Where A is the even_odd transform (generates a stream of even and odd numbers from input) and B and C are the basic transforms that receive even and odd input ''' cc = self.container assertions = self.assertTrue pubsub_cli = PubsubManagementServiceClient(node=cc.node) rr_cli = ResourceRegistryServiceClient(node=cc.node) tms_cli = TransformManagementServiceClient(node=cc.node) #------------------------------- # Process Definition #------------------------------- # Create the process definition for the basic transform process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample' } basic_transform_definition_id, _ = rr_cli.create(process_definition) # Create The process definition for the TransformEvenOdd process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformEvenOdd' } evenodd_transform_definition_id, _ = rr_cli.create(process_definition) #------------------------------- # Streams #------------------------------- streams = [pubsub_cli.create_stream() for i in xrange(5)] #------------------------------- # Subscriptions #------------------------------- query = StreamQuery(stream_ids=[streams[0]]) input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue') query = StreamQuery(stream_ids = [streams[1]]) # even output even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue') query = StreamQuery(stream_ids = [streams[2]]) # odd output odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue') #------------------------------- # Launch the EvenOdd Transform #------------------------------- evenodd_id = tms_cli.create_transform(name='even_odd', in_subscription_id=input_subscription_id, out_streams={'even':streams[1], 'odd':streams[2]}, process_definition_id=evenodd_transform_definition_id, configuration={}) tms_cli.activate_transform(evenodd_id) #------------------------------- # Launch the Even Processing Transform #------------------------------- even_transform_id = tms_cli.create_transform(name='even_transform', in_subscription_id = even_subscription_id, out_streams={'even_plus1':streams[3]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(even_transform_id) #------------------------------- # Launch the Odd Processing Transform #------------------------------- odd_transform_id = tms_cli.create_transform(name='odd_transform', in_subscription_id = odd_subscription_id, out_streams={'odd_plus1':streams[4]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(odd_transform_id) #------------------------------- # Set up final subscribers #------------------------------- evenplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[3]]), exchange_name='evenplus1_queue', name='EvenPlus1Subscription', description='EvenPlus1 SubscriptionDescription' ) oddplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[4]]), exchange_name='oddplus1_queue', name='OddPlus1Subscription', description='OddPlus1 SubscriptionDescription' ) total_msg_count = 2 msgs = gevent.queue.Queue() def even1_message_received(message, headers): input = int(message.get('num')) assertions( (input % 2) ) # Assert it is odd (transform adds 1) msgs.put(True) def odd1_message_received(message, headers): input = int(message.get('num')) assertions(not (input % 2)) # Assert it is even msgs.put(True) subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node) even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received) odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received) # Start subscribers even_subscriber.start() odd_subscriber.start() # Activate subscriptions pubsub_cli.activate_subscription(evenplus1_subscription_id) pubsub_cli.activate_subscription(oddplus1_subscription_id) #------------------------------- # Set up fake stream producer #------------------------------- pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node) stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0]) #------------------------------- # Start test #------------------------------- # Publish a stream for i in xrange(total_msg_count): stream_publisher.publish({'num':str(i)}) time.sleep(0.5) for i in xrange(total_msg_count * 2): try: msgs.get() except Empty: assertions(False, "Failed to process all messages correctly.") """
class TransformManagementServiceIntTest(IonIntegrationTestCase): def setUp(self): # set up the container self._start_container() self.container.start_rel_from_url("res/deploy/r2dm.yml") self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) self.tms_cli = TransformManagementServiceClient(node=self.container.node) self.rr_cli = ResourceRegistryServiceClient(node=self.container.node) self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node) self.input_stream_id = self.pubsub_cli.create_stream(name="input_stream", original=True) self.input_subscription_id = self.pubsub_cli.create_subscription( query=StreamQuery(stream_ids=[self.input_stream_id]), exchange_name="transform_input", name="input_subscription", ) self.output_stream_id = self.pubsub_cli.create_stream(name="output_stream", original=True) self.process_definition = ProcessDefinition(name="basic_transform_definition") self.process_definition.executable = { "module": "ion.processes.data.transforms.transform_example", "class": "TransformExample", } self.process_definition_id = self.procd_cli.create_process_definition( process_definition=self.process_definition ) @attr("LOCOINT") @unittest.skipIf(os.getenv("CEI_LAUNCH_TEST", False), "Not integrated for CEI") def test_create_transform(self): configuration = {"program_args": {"arg1": "value"}} transform_id = self.tms_cli.create_transform( name="test_transform", in_subscription_id=self.input_subscription_id, out_streams={"output": self.output_stream_id}, process_definition_id=self.process_definition_id, ) # test transform creation in rr transform = self.rr_cli.read(transform_id) self.assertEquals(transform.name, "test_transform") # test associations predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition] assocs = [] for p in predicates: assocs += self.rr_cli.find_associations(transform_id, p, id_only=True) self.assertEquals(len(assocs), 3) # test process creation transform = self.tms_cli.read_transform(transform_id) pid = transform.process_id proc = self.container.proc_manager.procs.get(pid) self.assertIsInstance(proc, TransformExample) # clean up self.tms_cli.delete_transform(transform_id) def test_create_transform_no_procdef(self): with self.assertRaises(NotFound): self.tms_cli.create_transform(name="test", in_subscription_id=self.input_subscription_id) def test_create_transform_bad_procdef(self): with self.assertRaises(NotFound): self.tms_cli.create_transform( name="test", in_subscription_id=self.input_subscription_id, process_definition_id="bad" ) def test_create_transform_no_config(self): transform_id = self.tms_cli.create_transform( name="test_transform", in_subscription_id=self.input_subscription_id, out_streams={"output": self.output_stream_id}, process_definition_id=self.process_definition_id, ) self.tms_cli.delete_transform(transform_id) def test_create_transform_name_failure(self): transform_id = self.tms_cli.create_transform( name="test_transform", in_subscription_id=self.input_subscription_id, out_streams={"output": self.output_stream_id}, process_definition_id=self.process_definition_id, ) with self.assertRaises(BadRequest): transform_id = self.tms_cli.create_transform( name="test_transform", in_subscription_id=self.input_subscription_id, out_streams={"output": self.output_stream_id}, process_definition_id=self.process_definition_id, ) self.tms_cli.delete_transform(transform_id) @attr("LOCOINT") @unittest.skipIf(os.getenv("CEI_LAUNCH_TEST", False), "Not integrated for CEI") def test_create_no_output(self): transform_id = self.tms_cli.create_transform( name="test_transform", in_subscription_id=self.input_subscription_id, process_definition_id=self.process_definition_id, ) predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition] assocs = [] for p in predicates: assocs += self.rr_cli.find_associations(transform_id, p, id_only=True) self.assertEquals(len(assocs), 2) # test process creation transform = self.tms_cli.read_transform(transform_id) pid = transform.process_id proc = self.container.proc_manager.procs.get(pid) self.assertIsInstance(proc, TransformExample) self.tms_cli.delete_transform(transform_id) def test_read_transform_exists(self): trans_obj = IonObject(RT.Transform, name="trans_obj") trans_id, _ = self.rr_cli.create(trans_obj) res = self.tms_cli.read_transform(trans_id) actual = self.rr_cli.read(trans_id) self.assertEquals(res._id, actual._id) def test_read_transform_nonexist(self): with self.assertRaises(NotFound) as e: self.tms_cli.read_transform("123") def test_activate_transform(self): transform_id = self.tms_cli.create_transform( name="test_transform", in_subscription_id=self.input_subscription_id, out_streams={"output": self.output_stream_id}, process_definition_id=self.process_definition_id, ) self.tms_cli.activate_transform(transform_id) # pubsub check if activated? self.tms_cli.delete_transform(transform_id) def test_activate_transform_nonexist(self): with self.assertRaises(NotFound): self.tms_cli.activate_transform("1234") def test_delete_transform(self): transform_id = self.tms_cli.create_transform( name="test_transform", in_subscription_id=self.input_subscription_id, process_definition_id=self.process_definition_id, ) self.tms_cli.delete_transform(transform_id) # assertions with self.assertRaises(NotFound): self.rr_cli.read(transform_id) def test_delete_transform_nonexist(self): with self.assertRaises(NotFound): self.tms_cli.delete_transform("123") def test_execute_transform(self): # set up process_definition = ProcessDefinition(name="procdef_execute") process_definition.executable["module"] = "ion.processes.data.transforms.transform_example" process_definition.executable["class"] = "ReverseTransform" data = [1, 2, 3] process_definition_id, _ = self.rr_cli.create(process_definition) retval = self.tms_cli.execute_transform(process_definition_id, data) self.assertEquals(retval, [3, 2, 1]) def test_integrated_transform(self): """ This example script runs a chained three way transform: B A < C Where A is the even_odd transform (generates a stream of even and odd numbers from input) and B and C are the basic transforms that receive even and odd input """ cc = self.container assertions = self.assertTrue pubsub_cli = PubsubManagementServiceClient(node=cc.node) rr_cli = ResourceRegistryServiceClient(node=cc.node) tms_cli = TransformManagementServiceClient(node=cc.node) # ------------------------------- # Process Definition # ------------------------------- # Create the process definition for the basic transform process_definition = IonObject(RT.ProcessDefinition, name="basic_transform_definition") process_definition.executable = { "module": "ion.processes.data.transforms.transform_example", "class": "TransformExample", } basic_transform_definition_id, _ = rr_cli.create(process_definition) # Create The process definition for the TransformEvenOdd process_definition = IonObject(RT.ProcessDefinition, name="evenodd_transform_definition") process_definition.executable = { "module": "ion.processes.data.transforms.transform_example", "class": "TransformEvenOdd", } evenodd_transform_definition_id, _ = rr_cli.create(process_definition) # ------------------------------- # Streams # ------------------------------- streams = [pubsub_cli.create_stream() for i in xrange(5)] # ------------------------------- # Subscriptions # ------------------------------- query = StreamQuery(stream_ids=[streams[0]]) input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name="input_queue") query = StreamQuery(stream_ids=[streams[1]]) # even output even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name="even_queue") query = StreamQuery(stream_ids=[streams[2]]) # odd output odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name="odd_queue") # ------------------------------- # Launch the EvenOdd Transform # ------------------------------- evenodd_id = tms_cli.create_transform( name="even_odd", in_subscription_id=input_subscription_id, out_streams={"even": streams[1], "odd": streams[2]}, process_definition_id=evenodd_transform_definition_id, configuration={}, ) tms_cli.activate_transform(evenodd_id) # ------------------------------- # Launch the Even Processing Transform # ------------------------------- even_transform_id = tms_cli.create_transform( name="even_transform", in_subscription_id=even_subscription_id, out_streams={"even_plus1": streams[3]}, process_definition_id=basic_transform_definition_id, configuration={}, ) tms_cli.activate_transform(even_transform_id) # ------------------------------- # Launch the Odd Processing Transform # ------------------------------- odd_transform_id = tms_cli.create_transform( name="odd_transform", in_subscription_id=odd_subscription_id, out_streams={"odd_plus1": streams[4]}, process_definition_id=basic_transform_definition_id, configuration={}, ) tms_cli.activate_transform(odd_transform_id) # ------------------------------- # Set up final subscribers # ------------------------------- evenplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[3]]), exchange_name="evenplus1_queue", name="EvenPlus1Subscription", description="EvenPlus1 SubscriptionDescription", ) oddplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[4]]), exchange_name="oddplus1_queue", name="OddPlus1Subscription", description="OddPlus1 SubscriptionDescription", ) total_msg_count = 2 msgs = gevent.queue.Queue() def even1_message_received(message, headers): input = int(message.get("num")) assertions((input % 2)) # Assert it is odd (transform adds 1) msgs.put(True) def odd1_message_received(message, headers): input = int(message.get("num")) assertions(not (input % 2)) # Assert it is even msgs.put(True) subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node) even_subscriber = subscriber_registrar.create_subscriber( exchange_name="evenplus1_queue", callback=even1_message_received ) odd_subscriber = subscriber_registrar.create_subscriber( exchange_name="oddplus1_queue", callback=odd1_message_received ) # Start subscribers even_subscriber.start() odd_subscriber.start() # Activate subscriptions pubsub_cli.activate_subscription(evenplus1_subscription_id) pubsub_cli.activate_subscription(oddplus1_subscription_id) # ------------------------------- # Set up fake stream producer # ------------------------------- pid = cc.spawn_process(name="dummy_process_for_test", module="pyon.ion.process", cls="SimpleProcess", config={}) dummy_process = cc.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node) stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0]) # ------------------------------- # Start test # ------------------------------- # Publish a stream for i in xrange(total_msg_count): stream_publisher.publish({"num": str(i)}) time.sleep(0.5) for i in xrange(total_msg_count * 2): try: msgs.get() except Empty: assertions(False, "Failed to process all messages correctly.") @attr("LOCOINT") @unittest.skipIf(os.getenv("CEI_LAUNCH_TEST", False), "CEI incompatible") def test_transform_restart(self): tms_cli = self.tms_cli rr_cli = self.rr_cli pubsub_cli = self.pubsub_cli cc = self.container assertions = self.assertTrue proc_def = ProcessDefinition( executable={ "module": "ion.services.dm.transformation.test.test_transform_service", "class": "TestTransform", } ) proc_def_id, _ = rr_cli.create(proc_def) stream_id = pubsub_cli.create_stream() subscription_id = pubsub_cli.create_subscription( query=StreamQuery(stream_ids=[stream_id]), exchange_name="test_transform" ) transform_id = tms_cli.create_transform( name="test_transform", in_subscription_id=subscription_id, process_definition_id=proc_def_id ) tms_cli.activate_transform(transform_id) transform = rr_cli.read(transform_id) pid = transform.process_id assertions(cc.proc_manager.procs.has_key(pid), "Transform process was never started.") cc.proc_manager.terminate_process(pid) assertions(not cc.proc_manager.procs.has_key(pid), "Transform process was never killed") handle = cc.proc_manager.procs_by_name["transform_management"] handle._restart_transform(transform_id) transform = rr_cli.read(transform_id) pid = transform.process_id assertions(cc.proc_manager.procs.has_key(pid), "Transform process was never restarted.")