def setUp(self): super(DataRetrieverServiceIntTest, self).setUp() self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.couch = self.container.datastore_manager.get_datastore( 'test_data_retriever', profile=DataStore.DS_PROFILE.SCIDATA) self.datastore_name = 'test_data_retriever' self.dr_cli = DataRetrieverServiceClient(node=self.container.node) self.dsm_cli = DatasetManagementServiceClient(node=self.container.node) self.rr_cli = ResourceRegistryServiceClient(node=self.container.node) self.ps_cli = PubsubManagementServiceClient(node=self.container.node) self.tms_cli = TransformManagementServiceClient( node=self.container.node) self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node) xs_dot_xp = CFG.core_xps.science_data try: self.XS, xp_base = xs_dot_xp.split('.') self.XP = '.'.join([bootstrap.get_sys_name(), xp_base]) except ValueError: raise StandardError( 'Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp) self.thread_pool = list()
def run_reverse_transform(self): ''' Runs a reverse transform example and displays the results of performing the transform ''' tms_cli = TransformManagementServiceClient(node=self.container.node) procd_cli = ProcessDispatcherServiceClient(node=self.container.node) #------------------------------- # Process Definition #------------------------------- process_definition = IonObject(RT.ProcessDefinition, name='transform_process_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class': 'ReverseTransform' } process_definition_id = procd_cli.create_process_definition( process_definition) #------------------------------- # Execute Transform #------------------------------- input = [1, 2, 3, 4] retval = tms_cli.execute_transform( process_definition_id=process_definition_id, data=[1, 2, 3, 4], configuration={}) log.debug('Transform Input: %s', input) log.debug('Transform Output: %s', retval)
def setUp(self): self._start_container() self.datastore_name = CACHE_DATASTORE_NAME self.container.start_rel_from_url('res/deploy/r2dm.yml') self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA) self.tms_cli = TransformManagementServiceClient() self.pubsub_cli = PubsubManagementServiceClient() self.pd_cli = ProcessDispatcherServiceClient() self.rr_cli = ResourceRegistryServiceClient() xs_dot_xp = CFG.core_xps.science_data try: self.XS, xp_base = xs_dot_xp.split('.') self.XP = '.'.join([bootstrap.get_sys_name(), xp_base]) except ValueError: raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
def on_start(self): super(CacheLauncher, self).on_start() tms_cli = TransformManagementServiceClient() pubsub_cli = PubsubManagementServiceClient() pd_cli = ProcessDispatcherServiceClient() dname = CACHE_DATASTORE_NAME number_of_workers = self.CFG.get_safe('process.number_of_workers', 2) proc_def = ProcessDefinition( name='last_update_worker_process', description= 'Worker process for caching the last update from a stream') proc_def.executable['module'] = 'ion.processes.data.last_update_cache' proc_def.executable['class'] = 'LastUpdateCache' proc_def_id = pd_cli.create_process_definition( process_definition=proc_def) xs_dot_xp = CFG.core_xps.science_data try: self.XS, xp_base = xs_dot_xp.split('.') self.XP = '.'.join([get_sys_name(), xp_base]) except ValueError: raise StandardError( 'Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp) subscription_id = pubsub_cli.create_subscription( query=ExchangeQuery(), exchange_name='last_update_cache') config = { 'couch_storage': { 'datastore_name': dname, 'datastore_profile': 'SCIDATA' } } for i in xrange(number_of_workers): transform_id = tms_cli.create_transform( name='last_update_cache%d' % i, description= 'last_update that compiles an aggregate of metadata', in_subscription_id=subscription_id, process_definition_id=proc_def_id, configuration=config) tms_cli.activate_transform(transform_id=transform_id)
def setUp(self): # set up the container self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) self.tms_cli = TransformManagementServiceClient(node=self.container.node) self.rr_cli = ResourceRegistryServiceClient(node=self.container.node) self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node) self.input_stream_id = self.pubsub_cli.create_stream(name='input_stream',original=True) self.input_subscription_id = self.pubsub_cli.create_subscription(query=StreamQuery(stream_ids=[self.input_stream_id]),exchange_name='transform_input',name='input_subscription') self.output_stream_id = self.pubsub_cli.create_stream(name='output_stream',original=True) self.process_definition = ProcessDefinition(name='basic_transform_definition') self.process_definition.executable = {'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample'} self.process_definition_id = self.procd_cli.create_process_definition(process_definition=self.process_definition)
def run_external_transform(self): ''' This example script illustrates how a transform can interact with the an outside process (very basic) it launches an external_transform example which uses the operating system command 'bc' to add 1 to the input Producer -> A -> 'FS.TEMP/transform_output' A is an external transform that spawns an OS process to increment the input by 1 ''' pubsub_cli = PubsubManagementServiceClient(node=self.container.node) tms_cli = TransformManagementServiceClient(node=self.container.node) procd_cli = ProcessDispatcherServiceClient(node=self.container.node) #------------------------------- # Process Definition #------------------------------- process_definition = ProcessDefinition( name='external_transform_definition') process_definition.executable[ 'module'] = 'ion.processes.data.transforms.transform_example' process_definition.executable['class'] = 'ExternalTransform' process_definition_id = procd_cli.create_process_definition( process_definition=process_definition) #------------------------------- # Streams #------------------------------- input_stream_id = pubsub_cli.create_stream(name='input_stream', original=True) #------------------------------- # Subscription #------------------------------- query = StreamQuery(stream_ids=[input_stream_id]) input_subscription_id = pubsub_cli.create_subscription( query=query, exchange_name='input_queue') #------------------------------- # Launch Transform #------------------------------- transform_id = tms_cli.create_transform( name='external_transform', in_subscription_id=input_subscription_id, process_definition_id=process_definition_id, configuration={}) tms_cli.activate_transform(transform_id) #------------------------------- # Launch Producer #------------------------------- id_p = self.container.spawn_process( 'myproducer', 'ion.processes.data.transforms.transform_example', 'TransformExampleProducer', { 'process': { 'type': 'stream_process', 'publish_streams': { 'out_stream': input_stream_id } }, 'stream_producer': { 'interval': 4000 } }) self.container.proc_manager.procs[id_p].start()
def run_even_odd_transform(self): ''' This example script runs a chained three way transform: B A < C Where A is the even_odd transform (generates a stream of even and odd numbers from input) and B and C are the basic transforms that receive even and odd input ''' pubsub_cli = PubsubManagementServiceClient(node=self.container.node) tms_cli = TransformManagementServiceClient(node=self.container.node) procd_cli = ProcessDispatcherServiceClient(node=self.container.node) #------------------------------- # Process Definition #------------------------------- # Create the process definition for the basic transform process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class': 'TransformExample' } basic_transform_definition_id = procd_cli.create_process_definition( process_definition=process_definition) # Create The process definition for the TransformEvenOdd process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class': 'TransformEvenOdd' } evenodd_transform_definition_id = procd_cli.create_process_definition( process_definition=process_definition) #------------------------------- # Streams #------------------------------- input_stream_id = pubsub_cli.create_stream(name='input_stream', original=True) even_stream_id = pubsub_cli.create_stream(name='even_stream', original=True) odd_stream_id = pubsub_cli.create_stream(name='odd_stream', original=True) #------------------------------- # Subscriptions #------------------------------- query = StreamQuery(stream_ids=[input_stream_id]) input_subscription_id = pubsub_cli.create_subscription( query=query, exchange_name='input_queue') query = StreamQuery(stream_ids=[even_stream_id]) even_subscription_id = pubsub_cli.create_subscription( query=query, exchange_name='even_queue') query = StreamQuery(stream_ids=[odd_stream_id]) odd_subscription_id = pubsub_cli.create_subscription( query=query, exchange_name='odd_queue') #------------------------------- # Launch the EvenOdd Transform #------------------------------- evenodd_id = tms_cli.create_transform( name='even_odd', in_subscription_id=input_subscription_id, out_streams={ 'even': even_stream_id, 'odd': odd_stream_id }, process_definition_id=evenodd_transform_definition_id, configuration={}) tms_cli.activate_transform(evenodd_id) #------------------------------- # Launch the Even Processing Transform #------------------------------- even_transform_id = tms_cli.create_transform( name='even_transform', in_subscription_id=even_subscription_id, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(even_transform_id) #------------------------------- # Launch the Odd Processing Transform #------------------------------- odd_transform_id = tms_cli.create_transform( name='odd_transform', in_subscription_id=odd_subscription_id, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(odd_transform_id) #------------------------------- # Spawn the Streaming Producer #------------------------------- id_p = self.container.spawn_process( 'myproducer', 'ion.processes.data.transforms.transform_example', 'TransformExampleProducer', { 'process': { 'type': 'stream_process', 'publish_streams': { 'out_stream': input_stream_id } }, 'stream_producer': { 'interval': 4000 } }) self.container.proc_manager.procs[id_p].start()
def run_basic_transform(self): ''' Runs a basic example of a transform. It chains two transforms together, each add 1 to their input Producer -> A -> B Producer generates a number every four seconds and publishes it on the 'ctd_output_stream' the producer is acting as a CTD or instrument in this example. A is a basic transform that increments its input and publishes it on the 'transform_output' stream. B is a basic transform that receives input. All transforms write logging data to 'FS.TEMP/transform_output' so you can visually see activity of the transforms ''' pubsub_cli = PubsubManagementServiceClient(node=self.container.node) tms_cli = TransformManagementServiceClient(node=self.container.node) procd_cli = ProcessDispatcherServiceClient(node=self.container.node) #------------------------------- # Process Definition #------------------------------- process_definition = IonObject(RT.ProcessDefinition, name='transform_process_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class': 'TransformExample' } process_definition_id = procd_cli.create_process_definition( process_definition) #------------------------------- # First Transform #------------------------------- # Create a dummy output stream from a 'ctd' instrument ctd_output_stream_id = pubsub_cli.create_stream( name='ctd_output_stream', original=True) # Create the subscription to the ctd_output_stream query = StreamQuery(stream_ids=[ctd_output_stream_id]) ctd_subscription_id = pubsub_cli.create_subscription( query=query, exchange_name='ctd_output') # Create an output stream for the transform transform_output_stream_id = pubsub_cli.create_stream( name='transform_output', original=True) configuration = {} # Launch the first transform process transform_id = tms_cli.create_transform( name='basic_transform', in_subscription_id=ctd_subscription_id, out_streams={'output': transform_output_stream_id}, process_definition_id=process_definition_id, configuration=configuration) tms_cli.activate_transform(transform_id) #------------------------------- # Second Transform #------------------------------- # Create a SUBSCRIPTION to this output stream for the second transform query = StreamQuery(stream_ids=[transform_output_stream_id]) second_subscription_id = pubsub_cli.create_subscription( query=query, exchange_name='final_output') # Create a final output stream final_output_id = pubsub_cli.create_stream(name='final_output', original=True) configuration = {} second_transform_id = tms_cli.create_transform( name='second_transform', in_subscription_id=second_subscription_id, out_streams={'output': final_output_id}, process_definition_id=process_definition_id, configuration=configuration) tms_cli.activate_transform(second_transform_id) #------------------------------- # Producer (Sample Input) #------------------------------- # Create a producing example process id_p = self.container.spawn_process( 'myproducer', 'ion.processes.data.transforms.transform_example', 'TransformExampleProducer', { 'process': { 'type': 'stream_process', 'publish_streams': { 'out_stream': ctd_output_stream_id } }, 'stream_producer': { 'interval': 4000 } }) self.container.proc_manager.procs[id_p].start()
def test_dm_integration(self): ''' test_salinity_transform Test full DM Services Integration ''' cc = self.container assertions = self.assertTrue #----------------------------- # Copy below here to run as a script (don't forget the imports of course!) #----------------------------- # Create some service clients... pubsub_management_service = PubsubManagementServiceClient(node=cc.node) ingestion_management_service = IngestionManagementServiceClient( node=cc.node) dataset_management_service = DatasetManagementServiceClient( node=cc.node) data_retriever_service = DataRetrieverServiceClient(node=cc.node) transform_management_service = TransformManagementServiceClient( node=cc.node) process_dispatcher = ProcessDispatcherServiceClient(node=cc.node) # declare some handy variables datastore_name = 'test_dm_integration' ### ### In the beginning there were two stream definitions... ### # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = pubsub_management_service.create_stream_definition( container=ctd_stream_def, name='Simulated CTD data') # create a stream definition for the data from the salinity Transform sal_stream_def_id = pubsub_management_service.create_stream_definition( container=SalinityTransform.outgoing_stream_def, name='Scalar Salinity data stream') ### ### And two process definitions... ### # one for the ctd simulator... producer_definition = ProcessDefinition() producer_definition.executable = { 'module': 'ion.processes.data.ctd_stream_publisher', 'class': 'SimpleCtdPublisher' } ctd_sim_procdef_id = process_dispatcher.create_process_definition( process_definition=producer_definition) # one for the salinity transform producer_definition = ProcessDefinition() producer_definition.executable = { 'module': 'ion.processes.data.transforms.ctd.ctd_L2_salinity', 'class': 'SalinityTransform' } salinity_transform_procdef_id = process_dispatcher.create_process_definition( process_definition=producer_definition) #--------------------------- # Set up ingestion - this is an operator concern - not done by SA in a deployed system #--------------------------- # Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile log.debug('Calling create_ingestion_configuration') ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration( exchange_point_id='science_data', couch_storage=CouchStorage(datastore_name=datastore_name, datastore_profile='SCIDATA'), number_of_workers=1) # ingestion_management_service.activate_ingestion_configuration( ingestion_configuration_id=ingestion_configuration_id) #--------------------------- # Set up the producer (CTD Simulator) #--------------------------- # Create the stream ctd_stream_id = pubsub_management_service.create_stream( stream_definition_id=ctd_stream_def_id) # Set up the datasets ctd_dataset_id = dataset_management_service.create_dataset( stream_id=ctd_stream_id, datastore_name=datastore_name, view_name='datasets/stream_join_granule') # Configure ingestion of this dataset ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration( dataset_id=ctd_dataset_id, archive_data=True, archive_metadata=True, ingestion_configuration_id= ingestion_configuration_id, # you need to know the ingestion configuration id! ) # Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service #--------------------------- # Set up the salinity transform #--------------------------- # Create the stream sal_stream_id = pubsub_management_service.create_stream( stream_definition_id=sal_stream_def_id) # Set up the datasets sal_dataset_id = dataset_management_service.create_dataset( stream_id=sal_stream_id, datastore_name=datastore_name, view_name='datasets/stream_join_granule') # Configure ingestion of the salinity as a dataset sal_dataset_config_id = ingestion_management_service.create_dataset_configuration( dataset_id=sal_dataset_id, archive_data=True, archive_metadata=True, ingestion_configuration_id= ingestion_configuration_id, # you need to know the ingestion configuration id! ) # Hold onto sal_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service # Create a subscription as input to the transform sal_transform_input_subscription_id = pubsub_management_service.create_subscription( query=StreamQuery(stream_ids=[ ctd_stream_id, ]), exchange_name='salinity_transform_input' ) # how do we make these names??? i.e. Should they be anonymous? # create the salinity transform sal_transform_id = transform_management_service.create_transform( name='example salinity transform', in_subscription_id=sal_transform_input_subscription_id, out_streams={ 'output': sal_stream_id, }, process_definition_id=salinity_transform_procdef_id, # no configuration needed at this time... ) # start the transform - for a test case it makes sense to do it before starting the producer but it is not required transform_management_service.activate_transform( transform_id=sal_transform_id) # Start the ctd simulator to produce some data configuration = { 'process': { 'stream_id': ctd_stream_id, } } ctd_sim_pid = process_dispatcher.schedule_process( process_definition_id=ctd_sim_procdef_id, configuration=configuration) ### ### Make a subscriber in the test to listen for salinity data ### salinity_subscription_id = pubsub_management_service.create_subscription( query=StreamQuery([ sal_stream_id, ]), exchange_name='salinity_test', name="test salinity subscription", ) pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node) result = gevent.event.AsyncResult() results = [] def message_received(message, headers): # Heads log.warn('Salinity data received!') results.append(message) if len(results) > 3: result.set(True) subscriber = subscriber_registrar.create_subscriber( exchange_name='salinity_test', callback=message_received) subscriber.start() # after the queue has been created it is safe to activate the subscription pubsub_management_service.activate_subscription( subscription_id=salinity_subscription_id) # Assert that we have received data assertions(result.get(timeout=10)) # stop the flow parse the messages... process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data for message in results: psd = PointSupplementStreamParser( stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message) # Test the handy info method for the names of fields in the stream def assertions('salinity' in psd.list_field_names()) # you have to know the name of the coverage in stream def salinity = psd.get_values('salinity') import numpy assertions(isinstance(salinity, numpy.ndarray)) assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
def test_integrated_transform(self): ''' This example script runs a chained three way transform: B A < C Where A is the even_odd transform (generates a stream of even and odd numbers from input) and B and C are the basic transforms that receive even and odd input ''' cc = self.container assertions = self.assertTrue pubsub_cli = PubsubManagementServiceClient(node=cc.node) rr_cli = ResourceRegistryServiceClient(node=cc.node) tms_cli = TransformManagementServiceClient(node=cc.node) #------------------------------- # Process Definition #------------------------------- # Create the process definition for the basic transform process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample' } basic_transform_definition_id, _ = rr_cli.create(process_definition) # Create The process definition for the TransformEvenOdd process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformEvenOdd' } evenodd_transform_definition_id, _ = rr_cli.create(process_definition) #------------------------------- # Streams #------------------------------- streams = [pubsub_cli.create_stream() for i in xrange(5)] #------------------------------- # Subscriptions #------------------------------- query = StreamQuery(stream_ids=[streams[0]]) input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue') query = StreamQuery(stream_ids = [streams[1]]) # even output even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue') query = StreamQuery(stream_ids = [streams[2]]) # odd output odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue') #------------------------------- # Launch the EvenOdd Transform #------------------------------- evenodd_id = tms_cli.create_transform(name='even_odd', in_subscription_id=input_subscription_id, out_streams={'even':streams[1], 'odd':streams[2]}, process_definition_id=evenodd_transform_definition_id, configuration={}) tms_cli.activate_transform(evenodd_id) #------------------------------- # Launch the Even Processing Transform #------------------------------- even_transform_id = tms_cli.create_transform(name='even_transform', in_subscription_id = even_subscription_id, out_streams={'even_plus1':streams[3]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(even_transform_id) #------------------------------- # Launch the Odd Processing Transform #------------------------------- odd_transform_id = tms_cli.create_transform(name='odd_transform', in_subscription_id = odd_subscription_id, out_streams={'odd_plus1':streams[4]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(odd_transform_id) #------------------------------- # Set up final subscribers #------------------------------- evenplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[3]]), exchange_name='evenplus1_queue', name='EvenPlus1Subscription', description='EvenPlus1 SubscriptionDescription' ) oddplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[4]]), exchange_name='oddplus1_queue', name='OddPlus1Subscription', description='OddPlus1 SubscriptionDescription' ) total_msg_count = 2 msgs = gevent.queue.Queue() def even1_message_received(message, headers): input = int(message.get('num')) assertions( (input % 2) ) # Assert it is odd (transform adds 1) msgs.put(True) def odd1_message_received(message, headers): input = int(message.get('num')) assertions(not (input % 2)) # Assert it is even msgs.put(True) subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node) even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received) odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received) # Start subscribers even_subscriber.start() odd_subscriber.start() # Activate subscriptions pubsub_cli.activate_subscription(evenplus1_subscription_id) pubsub_cli.activate_subscription(oddplus1_subscription_id) #------------------------------- # Set up fake stream producer #------------------------------- pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node) stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0]) #------------------------------- # Start test #------------------------------- # Publish a stream for i in xrange(total_msg_count): stream_publisher.publish({'num':str(i)}) time.sleep(0.5) for i in xrange(total_msg_count * 2): try: msgs.get() except Empty: assertions(False, "Failed to process all messages correctly.")
def test_usgs_integration(self): ''' test_usgs_integration Test full DM Services Integration using usgs ''' cc = self.container assertions = self.assertTrue #----------------------------- # Copy below here #----------------------------- pubsub_management_service = PubsubManagementServiceClient(node=cc.node) ingestion_management_service = IngestionManagementServiceClient(node=cc.node) dataset_management_service = DatasetManagementServiceClient(node=cc.node) data_retriever_service = DataRetrieverServiceClient(node=cc.node) transform_management_service = TransformManagementServiceClient(node=cc.node) process_dispatcher = ProcessDispatcherServiceClient(node=cc.node) process_list = [] datasets = [] datastore_name = 'test_usgs_integration' #--------------------------- # Set up ingestion #--------------------------- # Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile log.debug('Calling create_ingestion_configuration') ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration( exchange_point_id='science_data', couch_storage=CouchStorage(datastore_name=datastore_name,datastore_profile='SCIDATA'), number_of_workers=8 ) # ingestion_management_service.activate_ingestion_configuration( ingestion_configuration_id=ingestion_configuration_id) usgs_stream_def = USGS_stream_definition() stream_def_id = pubsub_management_service.create_stream_definition(container=usgs_stream_def, name='Junk definition') #--------------------------- # Set up the producers (CTD Simulators) #--------------------------- # Launch five simulated CTD producers for iteration in xrange(2): # Make a stream to output on stream_id = pubsub_management_service.create_stream(stream_definition_id=stream_def_id) #--------------------------- # Set up the datasets #--------------------------- dataset_id = dataset_management_service.create_dataset( stream_id=stream_id, datastore_name=datastore_name, view_name='datasets/stream_join_granule' ) # Keep track of the datasets datasets.append(dataset_id) stream_policy_id = ingestion_management_service.create_dataset_configuration( dataset_id = dataset_id, archive_data = True, archive_metadata = True, ingestion_configuration_id = ingestion_configuration_id ) producer_definition = ProcessDefinition() producer_definition.executable = { 'module':'eoi.agent.handler.usgs_stream_publisher', 'class':'UsgsPublisher' } configuration = { 'process':{ 'stream_id':stream_id, } } procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition) log.debug('LUKE_DEBUG: procdef_id: %s', procdef_id) pid = process_dispatcher.schedule_process(process_definition_id=procdef_id, configuration=configuration) # Keep track, we'll kill 'em later. process_list.append(pid) # Get about 4 seconds of data time.sleep(4) #--------------------------- # Stop producing data #--------------------------- for process in process_list: process_dispatcher.cancel_process(process) #---------------------------------------------- # The replay and the transform, a love story. #---------------------------------------------- # Happy Valentines to the clever coder who catches the above! transform_definition = ProcessDefinition() transform_definition.executable = { 'module':'ion.processes.data.transforms.transform_example', 'class':'TransformCapture' } transform_definition_id = process_dispatcher.create_process_definition(process_definition=transform_definition) dataset_id = datasets.pop() # Just need one for now replay_id, stream_id = data_retriever_service.define_replay(dataset_id=dataset_id) #-------------------------------------------- # I'm Selling magazine subscriptions here! #-------------------------------------------- subscription = pubsub_management_service.create_subscription(query=StreamQuery(stream_ids=[stream_id]), exchange_name='transform_capture_point') #-------------------------------------------- # Start the transform (capture) #-------------------------------------------- transform_id = transform_management_service.create_transform( name='capture_transform', in_subscription_id=subscription, process_definition_id=transform_definition_id ) transform_management_service.activate_transform(transform_id=transform_id) #-------------------------------------------- # BEGIN REPLAY! #-------------------------------------------- data_retriever_service.start_replay(replay_id=replay_id) #-------------------------------------------- # Lets get some boundaries #-------------------------------------------- bounds = dataset_management_service.get_dataset_bounds(dataset_id=dataset_id)