def run_reverse_transform(self):
''' Runs a reverse transform example and displays the results of performing the transform
'''
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = IonObject(RT.ProcessDefinition,
name='transform_process_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class': 'ReverseTransform'
}
process_definition_id = procd_cli.create_process_definition(
process_definition)
#-------------------------------
# Execute Transform
#-------------------------------
input = [1, 2, 3, 4]
retval = tms_cli.execute_transform(
process_definition_id=process_definition_id,
data=[1, 2, 3, 4],
configuration={})
log.debug('Transform Input: %s', input)
log.debug('Transform Output: %s', retval)
def run_reverse_transform(self):
''' Runs a reverse transform example and displays the results of performing the transform
'''
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = IonObject(RT.ProcessDefinition, name='transform_process_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'ReverseTransform'
}
process_definition_id = procd_cli.create_process_definition(process_definition)
#-------------------------------
# Execute Transform
#-------------------------------
input = [1,2,3,4]
retval = tms_cli.execute_transform(process_definition_id=process_definition_id,
data=[1,2,3,4],
configuration={})
log.debug('Transform Input: %s', input)
log.debug('Transform Output: %s', retval)
def run_external_transform(self):
'''
This example script illustrates how a transform can interact with the an outside process (very basic)
it launches an external_transform example which uses the operating system command 'bc' to add 1 to the input
Producer -> A -> 'FS.TEMP/transform_output'
A is an external transform that spawns an OS process to increment the input by 1
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = ProcessDefinition(name='external_transform_definition')
process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ExternalTransform'
process_definition_id = procd_cli.create_process_definition(process_definition=process_definition)
#-------------------------------
# Streams
#-------------------------------
input_stream_id = pubsub_cli.create_stream(name='input_stream', original=True)
#-------------------------------
# Subscription
#-------------------------------
query = StreamQuery(stream_ids=[input_stream_id])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
#-------------------------------
# Launch Transform
#-------------------------------
transform_id = tms_cli.create_transform(name='external_transform',
in_subscription_id=input_subscription_id,
process_definition_id=process_definition_id,
configuration={})
tms_cli.activate_transform(transform_id)
#-------------------------------
# Launch Producer
#-------------------------------
id_p = self.container.spawn_process('myproducer', 'ion.processes.data.transforms.transform_example', 'TransformExampleProducer', {'process':{'type':'stream_process','publish_streams':{'out_stream':input_stream_id}},'stream_producer':{'interval':4000}})
self.container.proc_manager.procs[id_p].start()
def setUp(self):
self._start_container()
self.datastore_name = CACHE_DATASTORE_NAME
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA)
self.tms_cli = TransformManagementServiceClient()
self.pubsub_cli = PubsubManagementServiceClient()
self.pd_cli = ProcessDispatcherServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([bootstrap.get_sys_name(), xp_base])
except ValueError:
raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
def setUp(self):
super(DataRetrieverServiceIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.couch = self.container.datastore_manager.get_datastore(
'test_data_retriever', profile=DataStore.DS_PROFILE.SCIDATA)
self.datastore_name = 'test_data_retriever'
self.dr_cli = DataRetrieverServiceClient(node=self.container.node)
self.dsm_cli = DatasetManagementServiceClient(node=self.container.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.container.node)
self.ps_cli = PubsubManagementServiceClient(node=self.container.node)
self.tms_cli = TransformManagementServiceClient(
node=self.container.node)
self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([bootstrap.get_sys_name(), xp_base])
except ValueError:
raise StandardError(
'Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure'
% xs_dot_xp)
self.thread_pool = list()
def setUp(self):
# set up the container
self._start_container()
self.cc = ContainerAgentClient(node=self.container.node,name=self.container.name)
self.cc.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.cc.node)
self.tms_cli = TransformManagementServiceClient(node=self.cc.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.cc.node)
self.input_stream = IonObject(RT.Stream,name='ctd1 output', description='output from a ctd')
self.input_stream.original = True
self.input_stream.mimetype = 'hdf'
self.input_stream_id = self.pubsub_cli.create_stream(self.input_stream)
self.input_subscription = IonObject(RT.Subscription,name='ctd1 subscription', description='subscribe to this if you want ctd1 data')
self.input_subscription.query['stream_id'] = self.input_stream_id
self.input_subscription.exchange_name = 'a queue'
self.input_subscription_id = self.pubsub_cli.create_subscription(self.input_subscription)
self.output_stream = IonObject(RT.Stream,name='transform output', description='output from the transform process')
self.output_stream.original = True
self.output_stream.mimetype='raw'
self.output_stream_id = self.pubsub_cli.create_stream(self.output_stream)
self.process_definition = IonObject(RT.ProcessDefinition,name='transform_process')
self.process_definition.executable = {'module': 'ion.services.dm.transformation.example.transform_example',
'class':'TransformExample'}
self.process_definition_id, _= self.rr_cli.create(self.process_definition)
def setUp(self):
# set up the container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2dm.yml")
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.tms_cli = TransformManagementServiceClient(node=self.container.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.container.node)
self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.input_stream_id = self.pubsub_cli.create_stream(name="input_stream", original=True)
self.input_subscription_id = self.pubsub_cli.create_subscription(
query=StreamQuery(stream_ids=[self.input_stream_id]),
exchange_name="transform_input",
name="input_subscription",
)
self.output_stream_id = self.pubsub_cli.create_stream(name="output_stream", original=True)
self.process_definition = ProcessDefinition(name="basic_transform_definition")
self.process_definition.executable = {
"module": "ion.processes.data.transforms.transform_example",
"class": "TransformExample",
}
self.process_definition_id = self.procd_cli.create_process_definition(
process_definition=self.process_definition
)
def on_start(self):
super(CacheLauncher, self).on_start()
tms_cli = TransformManagementServiceClient()
pubsub_cli = PubsubManagementServiceClient()
pd_cli = ProcessDispatcherServiceClient()
dname = CACHE_DATASTORE_NAME
number_of_workers = self.CFG.get_safe('process.number_of_workers', 2)
proc_def = ProcessDefinition(
name='last_update_worker_process',
description=
'Worker process for caching the last update from a stream')
proc_def.executable['module'] = 'ion.processes.data.last_update_cache'
proc_def.executable['class'] = 'LastUpdateCache'
proc_def_id = pd_cli.create_process_definition(
process_definition=proc_def)
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([get_sys_name(), xp_base])
except ValueError:
raise StandardError(
'Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure'
% xs_dot_xp)
subscription_id = pubsub_cli.create_subscription(
query=ExchangeQuery(), exchange_name='last_update_cache')
config = {
'couch_storage': {
'datastore_name': dname,
'datastore_profile': 'SCIDATA'
}
}
for i in xrange(number_of_workers):
transform_id = tms_cli.create_transform(
name='last_update_cache%d' % i,
description=
'last_update that compiles an aggregate of metadata',
in_subscription_id=subscription_id,
process_definition_id=proc_def_id,
configuration=config)
tms_cli.activate_transform(transform_id=transform_id)
def setUp(self):
# set up the container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.tms_cli = TransformManagementServiceClient(node=self.container.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.container.node)
self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.input_stream_id = self.pubsub_cli.create_stream(name='input_stream',original=True)
self.input_subscription_id = self.pubsub_cli.create_subscription(query=StreamQuery(stream_ids=[self.input_stream_id]),exchange_name='transform_input',name='input_subscription')
self.output_stream_id = self.pubsub_cli.create_stream(name='output_stream',original=True)
self.process_definition = ProcessDefinition(name='basic_transform_definition')
self.process_definition.executable = {'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'}
self.process_definition_id = self.procd_cli.create_process_definition(process_definition=self.process_definition)
def on_start(self):
super(CacheLauncher,self).on_start()
tms_cli = TransformManagementServiceClient()
pubsub_cli = PubsubManagementServiceClient()
pd_cli = ProcessDispatcherServiceClient()
dname = CACHE_DATASTORE_NAME
number_of_workers = self.CFG.get_safe('process.number_of_workers', 2)
proc_def = ProcessDefinition(name='last_update_worker_process',description='Worker process for caching the last update from a stream')
proc_def.executable['module'] = 'ion.processes.data.last_update_cache'
proc_def.executable['class'] = 'LastUpdateCache'
proc_def_id = pd_cli.create_process_definition(process_definition=proc_def)
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([get_sys_name(), xp_base])
except ValueError:
raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
subscription_id = pubsub_cli.create_subscription(query=ExchangeQuery(), exchange_name='last_update_cache')
config = {
'couch_storage' : {
'datastore_name' : dname,
'datastore_profile' : 'SCIDATA'
}
}
for i in xrange(number_of_workers):
transform_id = tms_cli.create_transform(
name='last_update_cache%d' % i,
description='last_update that compiles an aggregate of metadata',
in_subscription_id=subscription_id,
process_definition_id=proc_def_id,
configuration=config
)
tms_cli.activate_transform(transform_id=transform_id)
def setUp(self):
self._start_container()
self.datastore_name = CACHE_DATASTORE_NAME
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA)
self.tms_cli = TransformManagementServiceClient()
self.pubsub_cli = PubsubManagementServiceClient()
self.pd_cli = ProcessDispatcherServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([bootstrap.get_sys_name(), xp_base])
except ValueError:
raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, container=cc)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, container=cc)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
class TransformManagementServiceIntTest(IonIntegrationTestCase):
def setUp(self):
# set up the container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.tms_cli = TransformManagementServiceClient(node=self.container.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.container.node)
self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.input_stream_id = self.pubsub_cli.create_stream(name='input_stream',original=True)
self.input_subscription_id = self.pubsub_cli.create_subscription(query=StreamQuery(stream_ids=[self.input_stream_id]),exchange_name='transform_input',name='input_subscription')
self.output_stream_id = self.pubsub_cli.create_stream(name='output_stream',original=True)
self.process_definition = ProcessDefinition(name='basic_transform_definition')
self.process_definition.executable = {'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'}
self.process_definition_id = self.procd_cli.create_process_definition(process_definition=self.process_definition)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_create_transform(self):
configuration = {'program_args':{'arg1':'value'}}
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id)
# test transform creation in rr
transform = self.rr_cli.read(transform_id)
self.assertEquals(transform.name,'test_transform')
# test associations
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),3)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc,TransformExample)
# clean up
self.tms_cli.delete_transform(transform_id)
def test_create_transform_no_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',in_subscription_id=self.input_subscription_id)
def test_create_transform_bad_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',
in_subscription_id=self.input_subscription_id,
process_definition_id='bad')
def test_create_transform_no_config(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
def test_create_transform_name_failure(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
with self.assertRaises(BadRequest):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_create_no_output(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id,
)
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),2)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc,TransformExample)
self.tms_cli.delete_transform(transform_id)
def test_read_transform_exists(self):
trans_obj = IonObject(RT.Transform,name='trans_obj')
trans_id, _ = self.rr_cli.create(trans_obj)
res = self.tms_cli.read_transform(trans_id)
actual = self.rr_cli.read(trans_id)
self.assertEquals(res._id,actual._id)
def test_read_transform_nonexist(self):
with self.assertRaises(NotFound) as e:
self.tms_cli.read_transform('123')
def test_activate_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id
)
self.tms_cli.activate_transform(transform_id)
# pubsub check if activated?
self.tms_cli.delete_transform(transform_id)
def test_activate_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.activate_transform('1234')
def test_delete_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id
)
self.tms_cli.delete_transform(transform_id)
# assertions
with self.assertRaises(NotFound):
self.rr_cli.read(transform_id)
def test_delete_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.delete_transform('123')
def test_execute_transform(self):
# set up
process_definition = ProcessDefinition(name='procdef_execute')
process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ReverseTransform'
data = [1,2,3]
process_definition_id, _ = self.rr_cli.create(process_definition)
retval = self.tms_cli.execute_transform(process_definition_id,data)
self.assertEquals(retval,[3,2,1])
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, container=cc)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, container=cc)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
"""
def test_dm_integration(self):
'''
test_salinity_transform
Test full DM Services Integration
'''
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here to run as a script (don't forget the imports of course!)
#-----------------------------
# Create some service clients...
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(
node=cc.node)
dataset_management_service = DatasetManagementServiceClient(
node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
transform_management_service = TransformManagementServiceClient(
node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
# declare some handy variables
datastore_name = 'test_dm_integration'
###
### In the beginning there were two stream definitions...
###
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = pubsub_management_service.create_stream_definition(
container=ctd_stream_def, name='Simulated CTD data')
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = pubsub_management_service.create_stream_definition(
container=SalinityTransform.outgoing_stream_def,
name='Scalar Salinity data stream')
###
### And two process definitions...
###
# one for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.ctd_stream_publisher',
'class': 'SimpleCtdPublisher'
}
ctd_sim_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
# one for the salinity transform
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': 'ion.processes.data.transforms.ctd.ctd_L2_salinity',
'class': 'SalinityTransform'
}
salinity_transform_procdef_id = process_dispatcher.create_process_definition(
process_definition=producer_definition)
#---------------------------
# Set up ingestion - this is an operator concern - not done by SA in a deployed system
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,
datastore_profile='SCIDATA'),
number_of_workers=1)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
#---------------------------
# Set up the producer (CTD Simulator)
#---------------------------
# Create the stream
ctd_stream_id = pubsub_management_service.create_stream(
stream_definition_id=ctd_stream_def_id)
# Set up the datasets
ctd_dataset_id = dataset_management_service.create_dataset(
stream_id=ctd_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of this dataset
ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=ctd_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
#---------------------------
# Set up the salinity transform
#---------------------------
# Create the stream
sal_stream_id = pubsub_management_service.create_stream(
stream_definition_id=sal_stream_def_id)
# Set up the datasets
sal_dataset_id = dataset_management_service.create_dataset(
stream_id=sal_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule')
# Configure ingestion of the salinity as a dataset
sal_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id=sal_dataset_id,
archive_data=True,
archive_metadata=True,
ingestion_configuration_id=
ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto sal_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
# Create a subscription as input to the transform
sal_transform_input_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery(stream_ids=[
ctd_stream_id,
]),
exchange_name='salinity_transform_input'
) # how do we make these names??? i.e. Should they be anonymous?
# create the salinity transform
sal_transform_id = transform_management_service.create_transform(
name='example salinity transform',
in_subscription_id=sal_transform_input_subscription_id,
out_streams={
'output': sal_stream_id,
},
process_definition_id=salinity_transform_procdef_id,
# no configuration needed at this time...
)
# start the transform - for a test case it makes sense to do it before starting the producer but it is not required
transform_management_service.activate_transform(
transform_id=sal_transform_id)
# Start the ctd simulator to produce some data
configuration = {
'process': {
'stream_id': ctd_stream_id,
}
}
ctd_sim_pid = process_dispatcher.schedule_process(
process_definition_id=ctd_sim_procdef_id,
configuration=configuration)
###
### Make a subscriber in the test to listen for salinity data
###
salinity_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery([
sal_stream_id,
]),
exchange_name='salinity_test',
name="test salinity subscription",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process,
node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn('Salinity data received!')
results.append(message)
if len(results) > 3:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(
exchange_name='salinity_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
pubsub_management_service.activate_subscription(
subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=10))
# stop the flow parse the messages...
process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
for message in results:
psd = PointSupplementStreamParser(
stream_definition=SalinityTransform.outgoing_stream_def,
stream_granule=message)
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
import numpy
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) >
0.0) # salinity should always be greater than 0
class TransformManagementServiceIntTest(IonIntegrationTestCase):
def setUp(self):
# set up the container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2dm.yml")
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.tms_cli = TransformManagementServiceClient(node=self.container.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.container.node)
self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.input_stream_id = self.pubsub_cli.create_stream(name="input_stream", original=True)
self.input_subscription_id = self.pubsub_cli.create_subscription(
query=StreamQuery(stream_ids=[self.input_stream_id]),
exchange_name="transform_input",
name="input_subscription",
)
self.output_stream_id = self.pubsub_cli.create_stream(name="output_stream", original=True)
self.process_definition = ProcessDefinition(name="basic_transform_definition")
self.process_definition.executable = {
"module": "ion.processes.data.transforms.transform_example",
"class": "TransformExample",
}
self.process_definition_id = self.procd_cli.create_process_definition(
process_definition=self.process_definition
)
@attr("LOCOINT")
@unittest.skipIf(os.getenv("CEI_LAUNCH_TEST", False), "Not integrated for CEI")
def test_create_transform(self):
configuration = {"program_args": {"arg1": "value"}}
transform_id = self.tms_cli.create_transform(
name="test_transform",
in_subscription_id=self.input_subscription_id,
out_streams={"output": self.output_stream_id},
process_definition_id=self.process_definition_id,
)
# test transform creation in rr
transform = self.rr_cli.read(transform_id)
self.assertEquals(transform.name, "test_transform")
# test associations
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id, p, id_only=True)
self.assertEquals(len(assocs), 3)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc, TransformExample)
# clean up
self.tms_cli.delete_transform(transform_id)
def test_create_transform_no_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name="test", in_subscription_id=self.input_subscription_id)
def test_create_transform_bad_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(
name="test", in_subscription_id=self.input_subscription_id, process_definition_id="bad"
)
def test_create_transform_no_config(self):
transform_id = self.tms_cli.create_transform(
name="test_transform",
in_subscription_id=self.input_subscription_id,
out_streams={"output": self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
def test_create_transform_name_failure(self):
transform_id = self.tms_cli.create_transform(
name="test_transform",
in_subscription_id=self.input_subscription_id,
out_streams={"output": self.output_stream_id},
process_definition_id=self.process_definition_id,
)
with self.assertRaises(BadRequest):
transform_id = self.tms_cli.create_transform(
name="test_transform",
in_subscription_id=self.input_subscription_id,
out_streams={"output": self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
@attr("LOCOINT")
@unittest.skipIf(os.getenv("CEI_LAUNCH_TEST", False), "Not integrated for CEI")
def test_create_no_output(self):
transform_id = self.tms_cli.create_transform(
name="test_transform",
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id,
)
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id, p, id_only=True)
self.assertEquals(len(assocs), 2)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc, TransformExample)
self.tms_cli.delete_transform(transform_id)
def test_read_transform_exists(self):
trans_obj = IonObject(RT.Transform, name="trans_obj")
trans_id, _ = self.rr_cli.create(trans_obj)
res = self.tms_cli.read_transform(trans_id)
actual = self.rr_cli.read(trans_id)
self.assertEquals(res._id, actual._id)
def test_read_transform_nonexist(self):
with self.assertRaises(NotFound) as e:
self.tms_cli.read_transform("123")
def test_activate_transform(self):
transform_id = self.tms_cli.create_transform(
name="test_transform",
in_subscription_id=self.input_subscription_id,
out_streams={"output": self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.activate_transform(transform_id)
# pubsub check if activated?
self.tms_cli.delete_transform(transform_id)
def test_activate_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.activate_transform("1234")
def test_delete_transform(self):
transform_id = self.tms_cli.create_transform(
name="test_transform",
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
# assertions
with self.assertRaises(NotFound):
self.rr_cli.read(transform_id)
def test_delete_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.delete_transform("123")
def test_execute_transform(self):
# set up
process_definition = ProcessDefinition(name="procdef_execute")
process_definition.executable["module"] = "ion.processes.data.transforms.transform_example"
process_definition.executable["class"] = "ReverseTransform"
data = [1, 2, 3]
process_definition_id, _ = self.rr_cli.create(process_definition)
retval = self.tms_cli.execute_transform(process_definition_id, data)
self.assertEquals(retval, [3, 2, 1])
def test_integrated_transform(self):
"""
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
"""
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
# -------------------------------
# Process Definition
# -------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name="basic_transform_definition")
process_definition.executable = {
"module": "ion.processes.data.transforms.transform_example",
"class": "TransformExample",
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name="evenodd_transform_definition")
process_definition.executable = {
"module": "ion.processes.data.transforms.transform_example",
"class": "TransformEvenOdd",
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
# -------------------------------
# Streams
# -------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
# -------------------------------
# Subscriptions
# -------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name="input_queue")
query = StreamQuery(stream_ids=[streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name="even_queue")
query = StreamQuery(stream_ids=[streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name="odd_queue")
# -------------------------------
# Launch the EvenOdd Transform
# -------------------------------
evenodd_id = tms_cli.create_transform(
name="even_odd",
in_subscription_id=input_subscription_id,
out_streams={"even": streams[1], "odd": streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={},
)
tms_cli.activate_transform(evenodd_id)
# -------------------------------
# Launch the Even Processing Transform
# -------------------------------
even_transform_id = tms_cli.create_transform(
name="even_transform",
in_subscription_id=even_subscription_id,
out_streams={"even_plus1": streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={},
)
tms_cli.activate_transform(even_transform_id)
# -------------------------------
# Launch the Odd Processing Transform
# -------------------------------
odd_transform_id = tms_cli.create_transform(
name="odd_transform",
in_subscription_id=odd_subscription_id,
out_streams={"odd_plus1": streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={},
)
tms_cli.activate_transform(odd_transform_id)
# -------------------------------
# Set up final subscribers
# -------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name="evenplus1_queue",
name="EvenPlus1Subscription",
description="EvenPlus1 SubscriptionDescription",
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name="oddplus1_queue",
name="OddPlus1Subscription",
description="OddPlus1 SubscriptionDescription",
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get("num"))
assertions((input % 2)) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get("num"))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
even_subscriber = subscriber_registrar.create_subscriber(
exchange_name="evenplus1_queue", callback=even1_message_received
)
odd_subscriber = subscriber_registrar.create_subscriber(
exchange_name="oddplus1_queue", callback=odd1_message_received
)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
# -------------------------------
# Set up fake stream producer
# -------------------------------
pid = cc.spawn_process(name="dummy_process_for_test", module="pyon.ion.process", cls="SimpleProcess", config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
# -------------------------------
# Start test
# -------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({"num": str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
@attr("LOCOINT")
@unittest.skipIf(os.getenv("CEI_LAUNCH_TEST", False), "CEI incompatible")
def test_transform_restart(self):
tms_cli = self.tms_cli
rr_cli = self.rr_cli
pubsub_cli = self.pubsub_cli
cc = self.container
assertions = self.assertTrue
proc_def = ProcessDefinition(
executable={
"module": "ion.services.dm.transformation.test.test_transform_service",
"class": "TestTransform",
}
)
proc_def_id, _ = rr_cli.create(proc_def)
stream_id = pubsub_cli.create_stream()
subscription_id = pubsub_cli.create_subscription(
query=StreamQuery(stream_ids=[stream_id]), exchange_name="test_transform"
)
transform_id = tms_cli.create_transform(
name="test_transform", in_subscription_id=subscription_id, process_definition_id=proc_def_id
)
tms_cli.activate_transform(transform_id)
transform = rr_cli.read(transform_id)
pid = transform.process_id
assertions(cc.proc_manager.procs.has_key(pid), "Transform process was never started.")
cc.proc_manager.terminate_process(pid)
assertions(not cc.proc_manager.procs.has_key(pid), "Transform process was never killed")
handle = cc.proc_manager.procs_by_name["transform_management"]
handle._restart_transform(transform_id)
transform = rr_cli.read(transform_id)
pid = transform.process_id
assertions(cc.proc_manager.procs.has_key(pid), "Transform process was never restarted.")
def test_usgs_integration(self):
'''
test_usgs_integration
Test full DM Services Integration using usgs
'''
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here
#-----------------------------
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(node=cc.node)
dataset_management_service = DatasetManagementServiceClient(node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
transform_management_service = TransformManagementServiceClient(node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
process_list = []
datasets = []
datastore_name = 'test_usgs_integration'
#---------------------------
# Set up ingestion
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,datastore_profile='SCIDATA'),
number_of_workers=8
)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
usgs_stream_def = USGS_stream_definition()
stream_def_id = pubsub_management_service.create_stream_definition(container=usgs_stream_def, name='Junk definition')
#---------------------------
# Set up the producers (CTD Simulators)
#---------------------------
# Launch five simulated CTD producers
for iteration in xrange(2):
# Make a stream to output on
stream_id = pubsub_management_service.create_stream(stream_definition_id=stream_def_id)
#---------------------------
# Set up the datasets
#---------------------------
dataset_id = dataset_management_service.create_dataset(
stream_id=stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule'
)
# Keep track of the datasets
datasets.append(dataset_id)
stream_policy_id = ingestion_management_service.create_dataset_configuration(
dataset_id = dataset_id,
archive_data = True,
archive_metadata = True,
ingestion_configuration_id = ingestion_configuration_id
)
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':'eoi.agent.handler.usgs_stream_publisher',
'class':'UsgsPublisher'
}
configuration = {
'process':{
'stream_id':stream_id,
}
}
procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
log.debug('LUKE_DEBUG: procdef_id: %s', procdef_id)
pid = process_dispatcher.schedule_process(process_definition_id=procdef_id, configuration=configuration)
# Keep track, we'll kill 'em later.
process_list.append(pid)
# Get about 4 seconds of data
time.sleep(4)
#---------------------------
# Stop producing data
#---------------------------
for process in process_list:
process_dispatcher.cancel_process(process)
#----------------------------------------------
# The replay and the transform, a love story.
#----------------------------------------------
# Happy Valentines to the clever coder who catches the above!
transform_definition = ProcessDefinition()
transform_definition.executable = {
'module':'ion.processes.data.transforms.transform_example',
'class':'TransformCapture'
}
transform_definition_id = process_dispatcher.create_process_definition(process_definition=transform_definition)
dataset_id = datasets.pop() # Just need one for now
replay_id, stream_id = data_retriever_service.define_replay(dataset_id=dataset_id)
#--------------------------------------------
# I'm Selling magazine subscriptions here!
#--------------------------------------------
subscription = pubsub_management_service.create_subscription(query=StreamQuery(stream_ids=[stream_id]),
exchange_name='transform_capture_point')
#--------------------------------------------
# Start the transform (capture)
#--------------------------------------------
transform_id = transform_management_service.create_transform(
name='capture_transform',
in_subscription_id=subscription,
process_definition_id=transform_definition_id
)
transform_management_service.activate_transform(transform_id=transform_id)
#--------------------------------------------
# BEGIN REPLAY!
#--------------------------------------------
data_retriever_service.start_replay(replay_id=replay_id)
#--------------------------------------------
# Lets get some boundaries
#--------------------------------------------
bounds = dataset_management_service.get_dataset_bounds(dataset_id=dataset_id)
def run_external_transform(self):
'''
This example script illustrates how a transform can interact with the an outside process (very basic)
it launches an external_transform example which uses the operating system command 'bc' to add 1 to the input
Producer -> A -> 'FS.TEMP/transform_output'
A is an external transform that spawns an OS process to increment the input by 1
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = ProcessDefinition(
name='external_transform_definition')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ExternalTransform'
process_definition_id = procd_cli.create_process_definition(
process_definition=process_definition)
#-------------------------------
# Streams
#-------------------------------
input_stream_id = pubsub_cli.create_stream(name='input_stream',
original=True)
#-------------------------------
# Subscription
#-------------------------------
query = StreamQuery(stream_ids=[input_stream_id])
input_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='input_queue')
#-------------------------------
# Launch Transform
#-------------------------------
transform_id = tms_cli.create_transform(
name='external_transform',
in_subscription_id=input_subscription_id,
process_definition_id=process_definition_id,
configuration={})
tms_cli.activate_transform(transform_id)
#-------------------------------
# Launch Producer
#-------------------------------
id_p = self.container.spawn_process(
'myproducer', 'ion.processes.data.transforms.transform_example',
'TransformExampleProducer', {
'process': {
'type': 'stream_process',
'publish_streams': {
'out_stream': input_stream_id
}
},
'stream_producer': {
'interval': 4000
}
})
self.container.proc_manager.procs[id_p].start()
class LastUpdateCacheTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.datastore_name = CACHE_DATASTORE_NAME
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA)
self.tms_cli = TransformManagementServiceClient()
self.pubsub_cli = PubsubManagementServiceClient()
self.pd_cli = ProcessDispatcherServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([bootstrap.get_sys_name(), xp_base])
except ValueError:
raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
def make_points(self,definition,stream_id='I am very special', N=100):
from prototype.sci_data.constructor_apis import PointSupplementConstructor
import numpy as np
import random
definition.stream_resource_id = stream_id
total = N
n = 10 # at most n records per granule
i = 0
while i < total:
r = random.randint(1,n)
psc = PointSupplementConstructor(point_definition=definition, stream_id=stream_id)
for x in xrange(r):
i+=1
point_id = psc.add_point(time=i, location=(0,0,0))
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='temperature', value=np.random.normal(loc=48.0,scale=4.0, size=1)[0])
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='pressure', value=np.float32(1.0))
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='conductivity', value=np.float32(2.0))
granule = psc.close_stream_granule()
hdf_string = granule.identifiables[definition.data_stream_id].values
granule.identifiables[definition.data_stream_id].values = hdf_string
yield granule
return
def start_worker(self):
proc_def = ProcessDefinition()
proc_def.executable['module'] = 'ion.processes.data.last_update_cache'
proc_def.executable['class'] = 'LastUpdateCache'
proc_def_id = self.pd_cli.create_process_definition(process_definition=proc_def)
subscription_id = self.pubsub_cli.create_subscription(query=ExchangeQuery(), exchange_name='ingestion_aggregate')
config = {
'couch_storage' : {
'datastore_name' :self.datastore_name,
'datastore_profile' : 'SCIDATA'
}
}
transform_id = self.tms_cli.create_transform(
name='last_update_cache',
description='LastUpdate that compiles an aggregate of metadata',
in_subscription_id=subscription_id,
process_definition_id=proc_def_id,
configuration=config
)
self.tms_cli.activate_transform(transform_id=transform_id)
transform = self.rr_cli.read(transform_id)
pid = transform.process_id
handle = self.container.proc_manager.procs[pid]
return handle
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_last_update_cache(self):
handle = self.start_worker()
queue = Queue()
o_process = handle.process
def new_process(msg):
o_process(msg)
queue.put(True)
handle.process = new_process
definition = SBE37_CDM_stream_definition()
publisher = Publisher()
stream_def_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id)
time = float(0.0)
for granule in self.make_points(definition=definition, stream_id=stream_id, N=10):
publisher.publish(granule, to_name=(self.XP, stream_id+'.data'))
# Determinism sucks
try:
queue.get(timeout=5)
except Empty:
self.assertTrue(False, 'Process never received the message.')
doc = self.db.read(stream_id)
ntime = doc.variables['time'].value
self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.')
time = ntime
def test_dm_integration(self):
'''
test_salinity_transform
Test full DM Services Integration
'''
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here to run as a script (don't forget the imports of course!)
#-----------------------------
# Create some service clients...
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(node=cc.node)
dataset_management_service = DatasetManagementServiceClient(node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
transform_management_service = TransformManagementServiceClient(node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
# declare some handy variables
datastore_name = 'test_dm_integration'
###
### In the beginning there were two stream definitions...
###
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = pubsub_management_service.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data')
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = pubsub_management_service.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='Scalar Salinity data stream')
###
### And two process definitions...
###
# one for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':'ion.processes.data.ctd_stream_publisher',
'class':'SimpleCtdPublisher'
}
ctd_sim_procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
# one for the salinity transform
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':'ion.processes.data.transforms.ctd.ctd_L2_salinity',
'class':'SalinityTransform'
}
salinity_transform_procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
#---------------------------
# Set up ingestion - this is an operator concern - not done by SA in a deployed system
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,datastore_profile='SCIDATA'),
number_of_workers=1
)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
#---------------------------
# Set up the producer (CTD Simulator)
#---------------------------
# Create the stream
ctd_stream_id = pubsub_management_service.create_stream(stream_definition_id=ctd_stream_def_id)
# Set up the datasets
ctd_dataset_id = dataset_management_service.create_dataset(
stream_id=ctd_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule'
)
# Configure ingestion of this dataset
ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id = ctd_dataset_id,
archive_data = True,
archive_metadata = True,
ingestion_configuration_id = ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
#---------------------------
# Set up the salinity transform
#---------------------------
# Create the stream
sal_stream_id = pubsub_management_service.create_stream(stream_definition_id=sal_stream_def_id)
# Set up the datasets
sal_dataset_id = dataset_management_service.create_dataset(
stream_id=sal_stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule'
)
# Configure ingestion of the salinity as a dataset
sal_dataset_config_id = ingestion_management_service.create_dataset_configuration(
dataset_id = sal_dataset_id,
archive_data = True,
archive_metadata = True,
ingestion_configuration_id = ingestion_configuration_id, # you need to know the ingestion configuration id!
)
# Hold onto sal_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service
# Create a subscription as input to the transform
sal_transform_input_subscription_id = pubsub_management_service.create_subscription(
query = StreamQuery(stream_ids=[ctd_stream_id,]),
exchange_name='salinity_transform_input') # how do we make these names??? i.e. Should they be anonymous?
# create the salinity transform
sal_transform_id = transform_management_service.create_transform(
name='example salinity transform',
in_subscription_id=sal_transform_input_subscription_id,
out_streams={'output':sal_stream_id,},
process_definition_id = salinity_transform_procdef_id,
# no configuration needed at this time...
)
# start the transform - for a test case it makes sense to do it before starting the producer but it is not required
transform_management_service.activate_transform(transform_id=sal_transform_id)
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':ctd_stream_id,
}
}
ctd_sim_pid = process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration)
###
### Make a subscriber in the test to listen for salinity data
###
salinity_subscription_id = pubsub_management_service.create_subscription(
query=StreamQuery([sal_stream_id,]),
exchange_name = 'salinity_test',
name = "test salinity subscription",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn('Salinity data received!')
results.append(message)
if len(results) >3:
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='salinity_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
pubsub_management_service.activate_subscription(subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=10))
# stop the flow parse the messages...
process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
for message in results:
psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message)
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
import numpy
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
class TransformManagementServiceIntTest(IonIntegrationTestCase):
def setUp(self):
# set up the container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.tms_cli = TransformManagementServiceClient(node=self.container.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.container.node)
self.procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.input_stream_id = self.pubsub_cli.create_stream(name='input_stream',original=True)
self.input_subscription_id = self.pubsub_cli.create_subscription(query=StreamQuery(stream_ids=[self.input_stream_id]),exchange_name='transform_input',name='input_subscription')
self.output_stream_id = self.pubsub_cli.create_stream(name='output_stream',original=True)
self.process_definition = ProcessDefinition(name='basic_transform_definition')
self.process_definition.executable = {'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'}
self.process_definition_id = self.procd_cli.create_process_definition(process_definition=self.process_definition)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_create_transform(self):
configuration = {'program_args':{'arg1':'value'}}
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id)
# test transform creation in rr
transform = self.rr_cli.read(transform_id)
self.assertEquals(transform.name,'test_transform')
# test associations
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),3)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc,TransformExample)
# clean up
self.tms_cli.delete_transform(transform_id)
def test_create_transform_no_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',in_subscription_id=self.input_subscription_id)
def test_create_transform_bad_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',
in_subscription_id=self.input_subscription_id,
process_definition_id='bad')
def test_create_transform_no_config(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
def test_create_transform_name_failure(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
with self.assertRaises(BadRequest):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
self.tms_cli.delete_transform(transform_id)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
def test_create_no_output(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id,
)
predicates = [PRED.hasSubscription, PRED.hasOutStream, PRED.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),2)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs.get(pid)
self.assertIsInstance(proc,TransformExample)
self.tms_cli.delete_transform(transform_id)
def test_read_transform_exists(self):
trans_obj = IonObject(RT.Transform,name='trans_obj')
trans_id, _ = self.rr_cli.create(trans_obj)
res = self.tms_cli.read_transform(trans_id)
actual = self.rr_cli.read(trans_id)
self.assertEquals(res._id,actual._id)
def test_read_transform_nonexist(self):
with self.assertRaises(NotFound) as e:
self.tms_cli.read_transform('123')
def test_activate_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id
)
self.tms_cli.activate_transform(transform_id)
# pubsub check if activated?
self.tms_cli.delete_transform(transform_id)
def test_activate_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.activate_transform('1234')
def test_delete_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id
)
self.tms_cli.delete_transform(transform_id)
# assertions
with self.assertRaises(NotFound):
self.rr_cli.read(transform_id)
def test_delete_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.delete_transform('123')
def test_execute_transform(self):
# set up
process_definition = ProcessDefinition(name='procdef_execute')
process_definition.executable['module'] = 'ion.processes.data.transforms.transform_example'
process_definition.executable['class'] = 'ReverseTransform'
data = [1,2,3]
process_definition_id, _ = self.rr_cli.create(process_definition)
retval = self.tms_cli.execute_transform(process_definition_id,data)
self.assertEquals(retval,[3,2,1])
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
"""
class LastUpdateCacheTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.datastore_name = CACHE_DATASTORE_NAME
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.db = self.container.datastore_manager.get_datastore(self.datastore_name,DataStore.DS_PROFILE.SCIDATA)
self.tms_cli = TransformManagementServiceClient()
self.pubsub_cli = PubsubManagementServiceClient()
self.pd_cli = ProcessDispatcherServiceClient()
self.rr_cli = ResourceRegistryServiceClient()
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([bootstrap.get_sys_name(), xp_base])
except ValueError:
raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
def make_points(self,definition,stream_id='I am very special', N=100):
from prototype.sci_data.constructor_apis import PointSupplementConstructor
import numpy as np
import random
definition.stream_resource_id = stream_id
total = N
n = 10 # at most n records per granule
i = 0
while i < total:
r = random.randint(1,n)
psc = PointSupplementConstructor(point_definition=definition, stream_id=stream_id)
for x in xrange(r):
i+=1
point_id = psc.add_point(time=i, location=(0,0,0))
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='temperature', value=np.random.normal(loc=48.0,scale=4.0, size=1)[0])
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='pressure', value=np.float32(1.0))
psc.add_scalar_point_coverage(point_id=point_id, coverage_id='conductivity', value=np.float32(2.0))
granule = psc.close_stream_granule()
hdf_string = granule.identifiables[definition.data_stream_id].values
granule.identifiables[definition.data_stream_id].values = hdf_string
yield granule
return
def start_worker(self):
proc_def = ProcessDefinition()
proc_def.executable['module'] = 'ion.processes.data.last_update_cache'
proc_def.executable['class'] = 'LastUpdateCache'
proc_def_id = self.pd_cli.create_process_definition(process_definition=proc_def)
subscription_id = self.pubsub_cli.create_subscription(query=ExchangeQuery(), exchange_name='ingestion_aggregate')
config = {
'couch_storage' : {
'datastore_name' :self.datastore_name,
'datastore_profile' : 'SCIDATA'
}
}
transform_id = self.tms_cli.create_transform(
name='last_update_cache',
description='LastUpdate that compiles an aggregate of metadata',
in_subscription_id=subscription_id,
process_definition_id=proc_def_id,
configuration=config
)
self.tms_cli.activate_transform(transform_id=transform_id)
transform = self.rr_cli.read(transform_id)
pid = transform.process_id
handle = self.container.proc_manager.procs[pid]
return handle
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_last_update_cache(self):
handle = self.start_worker()
queue = Queue()
o_process = handle.process
def new_process(msg):
o_process(msg)
queue.put(True)
handle.process = new_process
definition = SBE37_CDM_stream_definition()
publisher = Publisher()
stream_def_id = self.pubsub_cli.create_stream_definition(container=definition)
stream_id = self.pubsub_cli.create_stream(stream_definition_id=stream_def_id)
time = float(0.0)
for granule in self.make_points(definition=definition, stream_id=stream_id, N=10):
publisher.publish(granule, to_name=(self.XP, stream_id+'.data'))
# Determinism sucks
try:
queue.get(timeout=5)
except Empty:
self.assertTrue(False, 'Process never received the message.')
doc = self.db.read(stream_id)
ntime = doc.variables['time'].value
self.assertTrue(ntime >= time, 'The documents did not sequentially get updated correctly.')
time = ntime
def test_integrated_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
cc = self.container
assertions = self.assertTrue
pubsub_cli = PubsubManagementServiceClient(node=cc.node)
rr_cli = ResourceRegistryServiceClient(node=cc.node)
tms_cli = TransformManagementServiceClient(node=cc.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id, _ = rr_cli.create(process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id, _ = rr_cli.create(process_definition)
#-------------------------------
# Streams
#-------------------------------
streams = [pubsub_cli.create_stream() for i in xrange(5)]
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[streams[0]])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [streams[1]]) # even output
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [streams[2]]) # odd output
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':streams[1], 'odd':streams[2]},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
out_streams={'even_plus1':streams[3]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
out_streams={'odd_plus1':streams[4]},
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Set up final subscribers
#-------------------------------
evenplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[3]]),
exchange_name='evenplus1_queue',
name='EvenPlus1Subscription',
description='EvenPlus1 SubscriptionDescription'
)
oddplus1_subscription_id = pubsub_cli.create_subscription(
query=StreamQuery([streams[4]]),
exchange_name='oddplus1_queue',
name='OddPlus1Subscription',
description='OddPlus1 SubscriptionDescription'
)
total_msg_count = 2
msgs = gevent.queue.Queue()
def even1_message_received(message, headers):
input = int(message.get('num'))
assertions( (input % 2) ) # Assert it is odd (transform adds 1)
msgs.put(True)
def odd1_message_received(message, headers):
input = int(message.get('num'))
assertions(not (input % 2)) # Assert it is even
msgs.put(True)
subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node)
even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received)
odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received)
# Start subscribers
even_subscriber.start()
odd_subscriber.start()
# Activate subscriptions
pubsub_cli.activate_subscription(evenplus1_subscription_id)
pubsub_cli.activate_subscription(oddplus1_subscription_id)
#-------------------------------
# Set up fake stream producer
#-------------------------------
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node)
stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0])
#-------------------------------
# Start test
#-------------------------------
# Publish a stream
for i in xrange(total_msg_count):
stream_publisher.publish({'num':str(i)})
time.sleep(0.5)
for i in xrange(total_msg_count * 2):
try:
msgs.get()
except Empty:
assertions(False, "Failed to process all messages correctly.")
def run_basic_transform(self):
''' Runs a basic example of a transform. It chains two transforms together, each add 1 to their input
Producer -> A -> B
Producer generates a number every four seconds and publishes it on the 'ctd_output_stream'
the producer is acting as a CTD or instrument in this example.
A is a basic transform that increments its input and publishes it on the 'transform_output' stream.
B is a basic transform that receives input.
All transforms write logging data to 'FS.TEMP/transform_output' so you can visually see activity of the transforms
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = IonObject(RT.ProcessDefinition,
name='transform_process_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class': 'TransformExample'
}
process_definition_id = procd_cli.create_process_definition(
process_definition)
#-------------------------------
# First Transform
#-------------------------------
# Create a dummy output stream from a 'ctd' instrument
ctd_output_stream_id = pubsub_cli.create_stream(
name='ctd_output_stream', original=True)
# Create the subscription to the ctd_output_stream
query = StreamQuery(stream_ids=[ctd_output_stream_id])
ctd_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='ctd_output')
# Create an output stream for the transform
transform_output_stream_id = pubsub_cli.create_stream(
name='transform_output', original=True)
configuration = {}
# Launch the first transform process
transform_id = tms_cli.create_transform(
name='basic_transform',
in_subscription_id=ctd_subscription_id,
out_streams={'output': transform_output_stream_id},
process_definition_id=process_definition_id,
configuration=configuration)
tms_cli.activate_transform(transform_id)
#-------------------------------
# Second Transform
#-------------------------------
# Create a SUBSCRIPTION to this output stream for the second transform
query = StreamQuery(stream_ids=[transform_output_stream_id])
second_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='final_output')
# Create a final output stream
final_output_id = pubsub_cli.create_stream(name='final_output',
original=True)
configuration = {}
second_transform_id = tms_cli.create_transform(
name='second_transform',
in_subscription_id=second_subscription_id,
out_streams={'output': final_output_id},
process_definition_id=process_definition_id,
configuration=configuration)
tms_cli.activate_transform(second_transform_id)
#-------------------------------
# Producer (Sample Input)
#-------------------------------
# Create a producing example process
id_p = self.container.spawn_process(
'myproducer', 'ion.processes.data.transforms.transform_example',
'TransformExampleProducer', {
'process': {
'type': 'stream_process',
'publish_streams': {
'out_stream': ctd_output_stream_id
}
},
'stream_producer': {
'interval': 4000
}
})
self.container.proc_manager.procs[id_p].start()
def run_basic_transform(self):
''' Runs a basic example of a transform. It chains two transforms together, each add 1 to their input
Producer -> A -> B
Producer generates a number every four seconds and publishes it on the 'ctd_output_stream'
the producer is acting as a CTD or instrument in this example.
A is a basic transform that increments its input and publishes it on the 'transform_output' stream.
B is a basic transform that receives input.
All transforms write logging data to 'FS.TEMP/transform_output' so you can visually see activity of the transforms
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
process_definition = IonObject(RT.ProcessDefinition, name='transform_process_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
process_definition_id = procd_cli.create_process_definition(process_definition)
#-------------------------------
# First Transform
#-------------------------------
# Create a dummy output stream from a 'ctd' instrument
ctd_output_stream_id = pubsub_cli.create_stream(name='ctd_output_stream', original=True)
# Create the subscription to the ctd_output_stream
query = StreamQuery(stream_ids=[ctd_output_stream_id])
ctd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='ctd_output')
# Create an output stream for the transform
transform_output_stream_id = pubsub_cli.create_stream(name='transform_output', original=True)
configuration = {}
# Launch the first transform process
transform_id = tms_cli.create_transform( name='basic_transform',
in_subscription_id=ctd_subscription_id,
out_streams={'output':transform_output_stream_id},
process_definition_id=process_definition_id,
configuration=configuration)
tms_cli.activate_transform(transform_id)
#-------------------------------
# Second Transform
#-------------------------------
# Create a SUBSCRIPTION to this output stream for the second transform
query = StreamQuery(stream_ids=[transform_output_stream_id])
second_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='final_output')
# Create a final output stream
final_output_id = pubsub_cli.create_stream(name='final_output', original=True)
configuration = {}
second_transform_id = tms_cli.create_transform( name='second_transform',
in_subscription_id=second_subscription_id,
out_streams={'output':final_output_id},
process_definition_id=process_definition_id,
configuration=configuration)
tms_cli.activate_transform(second_transform_id)
#-------------------------------
# Producer (Sample Input)
#-------------------------------
# Create a producing example process
id_p = self.container.spawn_process('myproducer', 'ion.processes.data.transforms.transform_example', 'TransformExampleProducer', {'process':{'type':'stream_process','publish_streams':{'out_stream':ctd_output_stream_id}},'stream_producer':{'interval':4000}})
self.container.proc_manager.procs[id_p].start()
def run_even_odd_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition,
name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class': 'TransformExample'
}
basic_transform_definition_id = procd_cli.create_process_definition(
process_definition=process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition,
name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class': 'TransformEvenOdd'
}
evenodd_transform_definition_id = procd_cli.create_process_definition(
process_definition=process_definition)
#-------------------------------
# Streams
#-------------------------------
input_stream_id = pubsub_cli.create_stream(name='input_stream',
original=True)
even_stream_id = pubsub_cli.create_stream(name='even_stream',
original=True)
odd_stream_id = pubsub_cli.create_stream(name='odd_stream',
original=True)
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[input_stream_id])
input_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids=[even_stream_id])
even_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids=[odd_stream_id])
odd_subscription_id = pubsub_cli.create_subscription(
query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(
name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={
'even': even_stream_id,
'odd': odd_stream_id
},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(
name='even_transform',
in_subscription_id=even_subscription_id,
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(
name='odd_transform',
in_subscription_id=odd_subscription_id,
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Spawn the Streaming Producer
#-------------------------------
id_p = self.container.spawn_process(
'myproducer', 'ion.processes.data.transforms.transform_example',
'TransformExampleProducer', {
'process': {
'type': 'stream_process',
'publish_streams': {
'out_stream': input_stream_id
}
},
'stream_producer': {
'interval': 4000
}
})
self.container.proc_manager.procs[id_p].start()
def run_even_odd_transform(self):
'''
This example script runs a chained three way transform:
B
A <
C
Where A is the even_odd transform (generates a stream of even and odd numbers from input)
and B and C are the basic transforms that receive even and odd input
'''
pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
tms_cli = TransformManagementServiceClient(node=self.container.node)
procd_cli = ProcessDispatcherServiceClient(node=self.container.node)
#-------------------------------
# Process Definition
#-------------------------------
# Create the process definition for the basic transform
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformExample'
}
basic_transform_definition_id = procd_cli.create_process_definition(process_definition=process_definition)
# Create The process definition for the TransformEvenOdd
process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition')
process_definition.executable = {
'module': 'ion.processes.data.transforms.transform_example',
'class':'TransformEvenOdd'
}
evenodd_transform_definition_id = procd_cli.create_process_definition(process_definition=process_definition)
#-------------------------------
# Streams
#-------------------------------
input_stream_id = pubsub_cli.create_stream(name='input_stream', original=True)
even_stream_id = pubsub_cli.create_stream(name='even_stream', original=True)
odd_stream_id = pubsub_cli.create_stream(name='odd_stream', original=True)
#-------------------------------
# Subscriptions
#-------------------------------
query = StreamQuery(stream_ids=[input_stream_id])
input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue')
query = StreamQuery(stream_ids = [even_stream_id])
even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue')
query = StreamQuery(stream_ids = [odd_stream_id])
odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue')
#-------------------------------
# Launch the EvenOdd Transform
#-------------------------------
evenodd_id = tms_cli.create_transform(name='even_odd',
in_subscription_id=input_subscription_id,
out_streams={'even':even_stream_id, 'odd':odd_stream_id},
process_definition_id=evenodd_transform_definition_id,
configuration={})
tms_cli.activate_transform(evenodd_id)
#-------------------------------
# Launch the Even Processing Transform
#-------------------------------
even_transform_id = tms_cli.create_transform(name='even_transform',
in_subscription_id = even_subscription_id,
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(even_transform_id)
#-------------------------------
# Launch the Odd Processing Transform
#-------------------------------
odd_transform_id = tms_cli.create_transform(name='odd_transform',
in_subscription_id = odd_subscription_id,
process_definition_id=basic_transform_definition_id,
configuration={})
tms_cli.activate_transform(odd_transform_id)
#-------------------------------
# Spawn the Streaming Producer
#-------------------------------
id_p = self.container.spawn_process('myproducer', 'ion.processes.data.transforms.transform_example', 'TransformExampleProducer', {'process':{'type':'stream_process','publish_streams':{'out_stream':input_stream_id}},'stream_producer':{'interval':4000}})
self.container.proc_manager.procs[id_p].start()
def test_usgs_integration(self):
'''
test_usgs_integration
Test full DM Services Integration using usgs
'''
cc = self.container
assertions = self.assertTrue
#-----------------------------
# Copy below here
#-----------------------------
pubsub_management_service = PubsubManagementServiceClient(node=cc.node)
ingestion_management_service = IngestionManagementServiceClient(node=cc.node)
dataset_management_service = DatasetManagementServiceClient(node=cc.node)
data_retriever_service = DataRetrieverServiceClient(node=cc.node)
transform_management_service = TransformManagementServiceClient(node=cc.node)
process_dispatcher = ProcessDispatcherServiceClient(node=cc.node)
process_list = []
datasets = []
datastore_name = 'test_usgs_integration'
#---------------------------
# Set up ingestion
#---------------------------
# Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile
log.debug('Calling create_ingestion_configuration')
ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=CouchStorage(datastore_name=datastore_name,datastore_profile='SCIDATA'),
number_of_workers=8
)
#
ingestion_management_service.activate_ingestion_configuration(
ingestion_configuration_id=ingestion_configuration_id)
usgs_stream_def = USGS_stream_definition()
stream_def_id = pubsub_management_service.create_stream_definition(container=usgs_stream_def, name='Junk definition')
#---------------------------
# Set up the producers (CTD Simulators)
#---------------------------
# Launch five simulated CTD producers
for iteration in xrange(2):
# Make a stream to output on
stream_id = pubsub_management_service.create_stream(stream_definition_id=stream_def_id)
#---------------------------
# Set up the datasets
#---------------------------
dataset_id = dataset_management_service.create_dataset(
stream_id=stream_id,
datastore_name=datastore_name,
view_name='datasets/stream_join_granule'
)
# Keep track of the datasets
datasets.append(dataset_id)
stream_policy_id = ingestion_management_service.create_dataset_configuration(
dataset_id = dataset_id,
archive_data = True,
archive_metadata = True,
ingestion_configuration_id = ingestion_configuration_id
)
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':'ion.agents.eoi.handler.usgs_stream_publisher',
'class':'UsgsPublisher'
}
configuration = {
'process':{
'stream_id':stream_id,
}
}
procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition)
log.debug('LUKE_DEBUG: procdef_id: %s', procdef_id)
pid = process_dispatcher.schedule_process(process_definition_id=procdef_id, configuration=configuration)
# Keep track, we'll kill 'em later.
process_list.append(pid)
# Get about 4 seconds of data
time.sleep(4)
#---------------------------
# Stop producing data
#---------------------------
for process in process_list:
process_dispatcher.cancel_process(process)
#----------------------------------------------
# The replay and the transform, a love story.
#----------------------------------------------
# Happy Valentines to the clever coder who catches the above!
transform_definition = ProcessDefinition()
transform_definition.executable = {
'module':'ion.processes.data.transforms.transform_example',
'class':'TransformCapture'
}
transform_definition_id = process_dispatcher.create_process_definition(process_definition=transform_definition)
dataset_id = datasets.pop() # Just need one for now
replay_id, stream_id = data_retriever_service.define_replay(dataset_id=dataset_id)
#--------------------------------------------
# I'm Selling magazine subscriptions here!
#--------------------------------------------
subscription = pubsub_management_service.create_subscription(query=StreamQuery(stream_ids=[stream_id]),
exchange_name='transform_capture_point')
#--------------------------------------------
# Start the transform (capture)
#--------------------------------------------
transform_id = transform_management_service.create_transform(
name='capture_transform',
in_subscription_id=subscription,
process_definition_id=transform_definition_id
)
transform_management_service.activate_transform(transform_id=transform_id)
#--------------------------------------------
# BEGIN REPLAY!
#--------------------------------------------
data_retriever_service.start_replay(replay_id=replay_id)
#--------------------------------------------
# Lets get some boundaries
#--------------------------------------------
bounds = dataset_management_service.get_dataset_bounds(dataset_id=dataset_id)
class TransformManagementServiceIntTest(IonIntegrationTestCase):
def setUp(self):
# set up the container
self._start_container()
self.cc = ContainerAgentClient(node=self.container.node,name=self.container.name)
self.cc.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub_cli = PubsubManagementServiceClient(node=self.cc.node)
self.tms_cli = TransformManagementServiceClient(node=self.cc.node)
self.rr_cli = ResourceRegistryServiceClient(node=self.cc.node)
self.input_stream = IonObject(RT.Stream,name='ctd1 output', description='output from a ctd')
self.input_stream.original = True
self.input_stream.mimetype = 'hdf'
self.input_stream_id = self.pubsub_cli.create_stream(self.input_stream)
self.input_subscription = IonObject(RT.Subscription,name='ctd1 subscription', description='subscribe to this if you want ctd1 data')
self.input_subscription.query['stream_id'] = self.input_stream_id
self.input_subscription.exchange_name = 'a queue'
self.input_subscription_id = self.pubsub_cli.create_subscription(self.input_subscription)
self.output_stream = IonObject(RT.Stream,name='transform output', description='output from the transform process')
self.output_stream.original = True
self.output_stream.mimetype='raw'
self.output_stream_id = self.pubsub_cli.create_stream(self.output_stream)
self.process_definition = IonObject(RT.ProcessDefinition,name='transform_process')
self.process_definition.executable = {'module': 'ion.services.dm.transformation.example.transform_example',
'class':'TransformExample'}
self.process_definition_id, _= self.rr_cli.create(self.process_definition)
def test_create_transform(self):
configuration = {'program_args':{'arg1':'value'}}
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id)
# test transform creation in rr
transform = self.rr_cli.read(transform_id)
self.assertEquals(transform.name,'test_transform')
# test associations
predicates = [AT.hasSubscription, AT.hasOutStream, AT.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),3)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs[pid]
self.assertTrue(isinstance(proc,TransformExample))
def test_create_transform_no_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',in_subscription_id=self.input_subscription_id)
def test_create_transform_bad_procdef(self):
with self.assertRaises(NotFound):
self.tms_cli.create_transform(name='test',
in_subscription_id=self.input_subscription_id,
process_definition_id='bad')
def test_create_transform_no_config(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
def test_create_transform_name_failure(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
with self.assertRaises(BadRequest):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id,
)
def test_create_no_output(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id,
)
predicates = [AT.hasSubscription, AT.hasOutStream, AT.hasProcessDefinition]
assocs = []
for p in predicates:
assocs += self.rr_cli.find_associations(transform_id,p,id_only=True)
self.assertEquals(len(assocs),2)
# test process creation
transform = self.tms_cli.read_transform(transform_id)
pid = transform.process_id
proc = self.container.proc_manager.procs[pid]
self.assertTrue(isinstance(proc,TransformExample))
def test_read_transform_exists(self):
trans_obj = IonObject(RT.Transform,name='trans_obj')
trans_id, _ = self.rr_cli.create(trans_obj)
res = self.tms_cli.read_transform(trans_id)
actual = self.rr_cli.read(trans_id)
self.assertEquals(res._id,actual._id)
def test_read_transform_nonexist(self):
with self.assertRaises(NotFound) as e:
res = self.tms_cli.read_transform('123')
def test_activate_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
out_streams={'output':self.output_stream_id},
process_definition_id=self.process_definition_id
)
self.tms_cli.activate_transform(transform_id)
# pubsub check if activated?
def test_activate_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.activate_transform('1234')
def test_delete_transform(self):
transform_id = self.tms_cli.create_transform(
name='test_transform',
in_subscription_id=self.input_subscription_id,
process_definition_id=self.process_definition_id
)
self.tms_cli.delete_transform(transform_id)
# assertions
with self.assertRaises(NotFound):
self.rr_cli.read(transform_id)
def test_delete_transform_nonexist(self):
with self.assertRaises(NotFound):
self.tms_cli.delete_transform('123')
