class TestDMEnd2End(IonIntegrationTestCase):
def setUp(self): # Love the non pep-8 convention
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.ingestion_management = IngestionManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.pids = []
self.event = Event()
self.exchange_space_name = 'test_granules'
self.exchange_point_name = 'science_data'
self.purge_queues()
self.queue_buffer = []
def purge_queues(self):
xn = self.container.ex_manager.create_xn_queue('science_granule_ingestion')
xn.purge()
def tearDown(self):
self.purge_queues()
for pid in self.pids:
self.container.proc_manager.terminate_process(pid)
IngestionManagementIntTest.clean_subscriptions()
for queue in self.queue_buffer:
if isinstance(queue, ExchangeNameQueue):
queue.delete()
elif isinstance(queue, str):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
def launch_producer(self, stream_id=''):
#--------------------------------------------------------------------------------
# Launch the producer
#--------------------------------------------------------------------------------
pid = self.container.spawn_process('better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process':{'stream_id':stream_id}})
self.pids.append(pid)
def get_ingestion_config(self):
#--------------------------------------------------------------------------------
# Grab the ingestion configuration from the resource registry
#--------------------------------------------------------------------------------
# The ingestion configuration should have been created by the bootstrap service
# which is configured through r2deploy.yml
ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration,id_only=True)
return ingest_configs[0]
def publish_hifi(self,stream_id,stream_route,offset=0):
pub = StandaloneStreamPublisher(stream_id, stream_route)
stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id)
stream_def_id = stream_def._id
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10) + (offset * 10)
rdt['temp'] = np.arange(10) + (offset * 10)
pub.publish(rdt.to_granule())
def publish_fake_data(self,stream_id, route):
for i in xrange(4):
self.publish_hifi(stream_id,route,i)
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def validate_granule_subscription(self, msg, route, stream_id):
if msg == {}:
return
rdt = RecordDictionaryTool.load_from_granule(msg)
log.info('%s', rdt.pretty_print())
self.assertIsInstance(msg,Granule,'Message is improperly formatted. (%s)' % type(msg))
self.event.set()
def make_file_data(self):
from interface.objects import File
import uuid
data = 'hello world\n'
rand = str(uuid.uuid4())[:8]
meta = File(name='/examples/' + rand + '.txt', group_id='example1')
return {'body': data, 'meta':meta}
def publish_file(self, stream_id, stream_route):
publisher = StandaloneStreamPublisher(stream_id,stream_route)
publisher.publish(self.make_file_data())
def wait_until_we_have_enough_granules(self, dataset_id='',granules=4):
datastore = self.get_datastore(dataset_id)
dataset = self.dataset_management.read_dataset(dataset_id)
with gevent.timeout.Timeout(40):
success = False
while not success:
success = len(datastore.query_view(dataset.view_name)) >= granules
gevent.sleep(0.1)
log.info(datastore.query_view(dataset.view_name))
def wait_until_we_have_enough_files(self):
datastore = self.container.datastore_manager.get_datastore('filesystem', DataStore.DS_PROFILE.FILESYSTEM)
now = time.time()
timeout = now + 10
done = False
while not done:
if now >= timeout:
raise Timeout('Files are not populating in time.')
if len(datastore.query_view('catalog/file_by_owner')) >= 1:
done = True
now = time.time()
def create_dataset(self, parameter_dict_id=''):
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
if not parameter_dict_id:
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
dataset_id = self.dataset_management.create_dataset('test_dataset', parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom)
return dataset_id
@unittest.skip('Doesnt work')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_replay_pause(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id)
dataset_id = self.create_dataset(pdict_id)
scov = DatasetManagementService._get_coverage(dataset_id)
bb = CoverageCraft(scov)
bb.rdt['time'] = np.arange(100)
bb.rdt['temp'] = np.random.random(100) + 30
bb.sync_with_granule()
DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations
# Set up the subscriber to verify the data
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
xp = self.container.ex_manager.create_xp('xp1')
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
# Set up the replay agent and the client wrapper
# 1) Define the Replay (dataset and stream to publish on)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream)
# 2) Make a client to the interact with the process (optionall provide it a process to bind with)
replay_client = ReplayClient(process_id)
# 3) Start the agent (launch the process)
self.data_retriever.start_replay_agent(self.replay_id)
# 4) Start replaying...
replay_client.start_replay()
# Wait till we get some granules
self.assertTrue(self.event.wait(5))
# We got granules, pause the replay, clear the queue and allow the process to finish consuming
replay_client.pause_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure there's no remaining messages being consumed
self.assertFalse(self.event.wait(1))
# Resume the replay and wait until we start getting granules again
replay_client.resume_replay()
self.assertTrue(self.event.wait(5))
# Stop the replay, clear the queues
replay_client.stop_replay()
gevent.sleep(1)
subscriber.xn.purge()
self.event.clear()
# Make sure that it did indeed stop
self.assertFalse(self.event.wait(1))
subscriber.stop()
@attr('SMOKE')
def test_dm_end_2_end(self):
#--------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
#--------------------------------------------------------------------------------
self.event.clear()
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
#--------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
#--------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
#--------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
#--------------------------------------------------------------------------------
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id,4)
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:])
self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all())
#--------------------------------------------------------------------------------
# Now to try the streamed approach
#--------------------------------------------------------------------------------
replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id)
log.info('Process ID: %s', process_id)
replay_client = ReplayClient(process_id)
#--------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
#--------------------------------------------------------------------------------
xp = self.container.ex_manager.create_xp(self.exchange_point_name)
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
self.queue_buffer.append(self.exchange_space_name)
subscriber.start()
subscriber.xn.bind(replay_route.routing_key, xp)
self.data_retriever.start_replay_agent(self.replay_id)
self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched')
replay_client.start_replay()
self.assertTrue(self.event.wait(10))
subscriber.stop()
self.data_retriever.cancel_replay_agent(self.replay_id)
#--------------------------------------------------------------------------------
# Test the slicing capabilities
#--------------------------------------------------------------------------------
granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)})
rdt = RecordDictionaryTool.load_from_granule(granule)
b = rdt['time'] == np.arange(5)
self.assertTrue(b.all() if not isinstance(b,bool) else b)
def test_retrieve_and_transform(self):
# Stream definition for the CTD data
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id)
ctd_stream_id, route = self.pubsub_management.create_stream('ctd stream', 'xp1', stream_definition_id=stream_def_id)
# Stream definition for the salinity data
salinity_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
sal_stream_def_id = self.pubsub_management.create_stream_definition('sal data', parameter_dictionary_id=salinity_pdict_id)
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
#--------------------------------------------------------------------------------
# Again with this ridiculous problem
#--------------------------------------------------------------------------------
self.get_datastore(dataset_id)
self.ingestion_management.persist_data_stream(stream_id=ctd_stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.random.randn(10) * 10 + 30
rdt['conductivity'] = np.random.randn(10) * 2 + 10
publisher = StandaloneStreamPublisher(ctd_stream_id, route)
publisher.publish(rdt.to_granule())
rdt['time'] = np.arange(10,20)
publisher.publish(rdt.to_granule())
self.wait_until_we_have_enough_granules(dataset_id, 2)
granule = self.data_retriever.retrieve(dataset_id,
None,
None,
'ion.processes.data.transforms.ctd.ctd_L2_salinity',
'CTDL2SalinityTransformAlgorithm',
kwargs=dict(params=sal_stream_def_id))
rdt = RecordDictionaryTool.load_from_granule(granule)
for i in rdt['salinity']:
self.assertNotEquals(i,0)
def test_last_granule(self):
#--------------------------------------------------------------------------------
# Create the necessary configurations for the test
#--------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('last_granule', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id)
#--------------------------------------------------------------------------------
# Create the datastore first,
#--------------------------------------------------------------------------------
self.get_datastore(dataset_id)
self.publish_hifi(stream_id,route, 0)
self.publish_hifi(stream_id,route, 1)
self.wait_until_we_have_enough_granules(dataset_id,2) # I just need two
success = False
def verifier():
replay_granule = self.data_retriever.retrieve_last_granule(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(10) + 10
if not isinstance(comp,bool):
return comp.all()
return False
success = poll(verifier)
self.assertTrue(success)
success = False
def verify_points():
replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id,5)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(15,20)
if not isinstance(comp,bool):
return comp.all()
return False
success = poll(verify_points)
self.assertTrue(success)
def test_replay_with_parameters(self):
#--------------------------------------------------------------------------------
# Create the configurations and the dataset
#--------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id)
#--------------------------------------------------------------------------------
# Coerce the datastore into existence (beats race condition)
#--------------------------------------------------------------------------------
self.get_datastore(dataset_id)
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id,4)
query = {
'start_time': 0,
'end_time': 20,
'stride_time' : 2,
'parameters': ['time','temp']
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
comp = np.arange(0,20,2) == rdt['time']
self.assertTrue(comp.all(),'%s' % rdt.pretty_print())
self.assertEquals(set(rdt.iterkeys()), set(['time','temp']))
extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp'])
self.assertTrue(extents['time']>=20)
self.assertTrue(extents['temp']>=20)
def test_repersist_data(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition(name='ctd', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream(name='repersist', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id)
self.get_datastore(dataset_id)
self.publish_hifi(stream_id,route,0)
self.publish_hifi(stream_id,route,1)
self.wait_until_we_have_enough_granules(dataset_id,2)
self.ingestion_management.unpersist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id,dataset_id=dataset_id)
self.publish_hifi(stream_id,route,2)
self.publish_hifi(stream_id,route,3)
self.wait_until_we_have_enough_granules(dataset_id,4)
success = False
with gevent.timeout.Timeout(5):
while not success:
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(0,40)
if not isinstance(comp,bool):
success = comp.all()
gevent.sleep(1)
self.assertTrue(success)
class VisualizationIntegrationTestHelper(IonIntegrationTestCase):
def create_ctd_input_stream_and_data_product(self, data_product_name='ctd_parsed'):
cc = self.container
assertions = self.assertTrue
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
self.vis_client = VisualizationServiceClient(node=self.container.node)
#-------------------------------
# Create CTD Parsed as the initial data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data')
log.debug('Creating new CDM data product with a stream definition')
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name=data_product_name,
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product_id = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug('new ctd_parsed_data_product_id = %s' % ctd_parsed_data_product_id)
#Only ever need one device for testing purposes.
instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='SBE37IMDevice')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
ctd_stream_id = stream_ids[0]
return ctd_stream_id, ctd_parsed_data_product_id
def create_data_product(self, dp_name = "", dp_description = ""):
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters() # this creates a ParameterDictionary object
parameter_dictionary = parameter_dictionary.dump() # this returns a python dictionary
data_prod_obj = IonObject(RT.DataProduct,
name=dp_name,
description=dp_description,
temporal_domain = tdom,
spatial_domain = sdom)
data_prod_id = self.create_data_product(data_prod_obj, stream_definition_id, parameter_dictionary)
return data_prod_id, data_prod_obj
def start_simple_input_stream_process(self, ctd_stream_id):
return self.start_input_stream_process(ctd_stream_id)
def start_sinusoidal_input_stream_process(self, ctd_stream_id):
return self.start_input_stream_process(ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher', 'SinusoidalCtdPublisher')
def start_input_stream_process(self, ctd_stream_id, module = 'ion.processes.data.ctd_stream_publisher', class_name= 'SimpleCtdPublisher'):
###
### Start the process for producing the CTD data
###
# process definition for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':module,
'class':class_name
}
ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(process_definition=producer_definition)
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':ctd_stream_id,
}
}
ctd_sim_pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration)
return ctd_sim_pid
def start_output_stream_and_listen(self, ctd_stream_id, data_product_stream_ids, message_count_per_stream=10):
cc = self.container
assertions = self.assertTrue
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name = 'workflow_test',
exchange_point = 'science_data',
name = "test workflow transformations",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, container=cc)
result = gevent.event.AsyncResult()
results = []
message_count = len(data_product_stream_ids) * message_count_per_stream
def message_received(message, headers):
# Heads
results.append(message)
if len(results) >= message_count: #Only wait for so many messages - per stream
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='workflow_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
#Start the input stream process
if ctd_stream_id is not None:
ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id)
# Assert that we have received data
assertions(result.get(timeout=30))
# stop the flow parse the messages...
if ctd_stream_id is not None:
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def validate_messages(self, results):
cc = self.container
assertions = self.assertTrue
first_salinity_values = None
for message in results:
rdt = RecordDictionaryTool.load_from_granule(message)
try:
temp = get_safe(rdt, 'temp')
# psd = PointSupplementStreamParser(stream_definition=self.ctd_stream_def, stream_granule=message)
# temp = psd.get_values('temperature')
# log.info(psd.list_field_names())
except KeyError as ke:
temp = None
if temp is not None:
assertions(isinstance(temp, numpy.ndarray))
log.info( 'temperature=' + str(numpy.nanmin(temp)))
first_salinity_values = None
else:
#psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message)
#log.info( psd.list_field_names())
# Test the handy info method for the names of fields in the stream def
#assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = get_safe(rdt, 'salinity')
#salinity = psd.get_values('salinity')
log.info( 'salinity=' + str(numpy.nanmin(salinity)))
# Check to see if salinity has values
assertions(salinity != None)
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
if first_salinity_values is None:
first_salinity_values = salinity.tolist()
else:
second_salinity_values = salinity.tolist()
assertions(len(first_salinity_values) == len(second_salinity_values))
for idx in range(0,len(first_salinity_values)):
assertions(first_salinity_values[idx]*2.0 == second_salinity_values[idx])
def validate_data_ingest_retrieve(self, dataset_id):
assertions = self.assertTrue
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
#validate that data was ingested
replay_granule = self.data_retriever.retrieve_last_granule(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
salinity = get_safe(rdt, 'salinity')
assertions(salinity != None)
#retrieve all the granules from the database and check the values
replay_granule_all = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule_all)
for k, v in rdt.iteritems():
if k == 'salinity':
for val in numpy.nditer(v):
assertions(val > 0)
def create_salinity_data_process_definition(self):
# Salinity: Data Process Definition
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='ctd_salinity')
if len(dpd) > 0:
return dpd[0]
log.debug("Create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_salinity',
description='create a salinity data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Excpetion as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='Salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(sal_stream_def_id, ctd_L2_salinity_dprocdef_id )
return ctd_L2_salinity_dprocdef_id
def create_salinity_doubler_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler')
if len(dpd) > 0:
return dpd[0]
# Salinity Doubler: Data Process Definition
log.debug("Create data process definition SalinityDoublerTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='salinity_doubler',
description='create a salinity doubler data product',
module='ion.processes.data.transforms.example_double_salinity',
class_name='SalinityDoubler',
process_source='SalinityDoubler source code here...')
try:
salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new SalinityDoubler data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(salinity_double_stream_def_id, salinity_doubler_dprocdef_id )
return salinity_doubler_dprocdef_id
def create_transform_process(self, data_process_definition_id, data_process_input_dp_id):
data_process_definition = self.rrclient.read(data_process_definition_id)
# Find the link between the output Stream Definition resource and the Data Process Definition resource
stream_ids,_ = self.rrclient.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True)
if not stream_ids:
raise Inconsistent("The data process definition %s is missing an association to an output stream definition" % data_process_definition._id )
process_output_stream_def_id = stream_ids[0]
#Concatenate the name of the workflow and data process definition for the name of the data product output
data_process_name = data_process_definition.name
# Create the output data product of the transform
transform_dp_obj = IonObject(RT.DataProduct, name=data_process_name,description=data_process_definition.description)
transform_dp_id = self.dataproductclient.create_data_product(transform_dp_obj, process_output_stream_def_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=transform_dp_id)
#last one out of the for loop is the output product id
output_data_product_id = transform_dp_id
# Create the transform data process
log.debug("create data_process and start it")
data_process_id = self.dataprocessclient.create_data_process(data_process_definition._id, [data_process_input_dp_id], {'output':transform_dp_id})
self.dataprocessclient.activate_data_process(data_process_id)
#Find the id of the output data stream
stream_ids, _ = self.rrclient.find_objects(transform_dp_id, PRED.hasStream, None, True)
if not stream_ids:
raise Inconsistent("The data process %s is missing an association to an output stream" % data_process_id )
return data_process_id, output_data_product_id
def create_google_dt_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform')
if len(dpd) > 0:
return dpd[0]
# Data Process Definition
log.debug("Create data process definition GoogleDtTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='google_dt_transform',
description='Convert data streams to Google DataTables',
module='ion.processes.data.transforms.viz.google_dt',
class_name='VizTransformGoogleDT',
process_source='VizTransformGoogleDT source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new VizTransformGoogleDT data process definition: %s" %ex)
# create a stream definition for the data from the
stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformGoogleDT.outgoing_stream_def, name='VizTransformGoogleDT')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id )
return procdef_id
def validate_google_dt_transform_results(self, results):
cc = self.container
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results,Granule):
results =[results]
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
gdt_data = get_safe(rdt, 'google_dt_components')
# IF this granule does not contains google dt, skip
if gdt_data == None:
continue
gdt = gdt_data[0]
assertions(gdt['viz_product_type'] == 'google_dt' )
assertions(len(gdt['data_description']) >= 0) # Need to come up with a better check
assertions(len(gdt['data_content']) >= 0)
def create_mpl_graphs_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform')
if len(dpd) > 0:
return dpd[0]
#Data Process Definition
log.debug("Create data process definition MatplotlibGraphsTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='mpl_graphs_transform',
description='Convert data streams to Matplotlib graphs',
module='ion.processes.data.transforms.viz.matplotlib_graphs',
class_name='VizTransformMatplotlibGraphs',
process_source='VizTransformMatplotlibGraphs source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new VizTransformMatplotlibGraphs data process definition: %s" %ex)
# create a stream definition for the data
stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformMatplotlibGraphs.outgoing_stream_def, name='VizTransformMatplotlibGraphs')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id )
return procdef_id
def validate_mpl_graphs_transform_results(self, results):
cc = self.container
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results,Granule):
results =[results]
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
graphs = get_safe(rdt, 'matplotlib_graphs')
if graphs == None:
continue
for graph in graphs[0]:
# At this point only dictionaries containing image data should be passed
# For some reason non dictionary values are filtering through.
if not isinstance(graph, dict):
continue
assertions(graph['viz_product_type'] == 'matplotlib_graphs' )
# check to see if the list (numpy array) contains actual images
assertions(imghdr.what(graph['image_name'], h = graph['image_obj']) == 'png')
def validate_vis_service_google_dt_results(self, results):
assertions = self.assertTrue
assertions(results)
gdt_str = (results.lstrip("google.visualization.Query.setResponse(")).rstrip(")")
assertions(len(gdt_str) > 0)
return
def validate_vis_service_mpl_graphs_results(self, results):
assertions = self.assertTrue
assertions(results)
# check to see if the object passed is a dictionary with a valid image object in it
image_format = results["content_type"].lstrip("image/")
assertions(imghdr.what(results['image_name'], h = base64.decodestring(results['image_obj'])) == image_format)
return
class TestDMEnd2End(IonIntegrationTestCase):
def setUp(self): # Love the non pep-8 convention
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.ingestion_management = IngestionManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.pids = []
self.event = Event()
self.exchange_space_name = "test_granules"
self.exchange_point_name = "science_data"
self.purge_queues()
def purge_queues(self):
xn = self.container.ex_manager.create_xn_queue("science_granule_ingestion")
xn.purge()
def tearDown(self):
self.purge_queues()
for pid in self.pids:
self.process_dispatcher.cancel_process(pid)
IngestionManagementIntTest.clean_subscriptions()
def launch_producer(self, stream_id=""):
# --------------------------------------------------------------------------------
# Create the process definition for the producer
# --------------------------------------------------------------------------------
producer_definition = ProcessDefinition(name="Example Data Producer")
producer_definition.executable = {
"module": "ion.processes.data.example_data_producer",
"class": "BetterDataProducer",
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=producer_definition
)
# --------------------------------------------------------------------------------
# Launch the producer
# --------------------------------------------------------------------------------
config = DotDict()
config.process.stream_id = stream_id
pid = self.process_dispatcher.schedule_process(
process_definition_id=process_definition_id, configuration=config
)
self.pids.append(pid)
def get_ingestion_config(self):
# --------------------------------------------------------------------------------
# Grab the ingestion configuration from the resource registry
# --------------------------------------------------------------------------------
# The ingestion configuration should have been created by the bootstrap service
# which is configured through r2deploy.yml
ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration, id_only=True)
return ingest_configs[0]
def publish_hifi(self, stream_id, offset=0):
pub = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id)
black_box = CoverageCraft()
black_box.rdt["time"] = np.arange(10) + (offset * 10)
black_box.rdt["temp"] = (np.arange(10) + (offset * 10)) * 2
granule = black_box.to_granule()
pub.publish(granule)
def publish_fake_data(self, stream_id):
for i in xrange(4):
self.publish_hifi(stream_id, i)
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def validate_granule_subscription(self, msg, header):
if msg == {}:
return
self.assertIsInstance(msg, Granule, "Message is improperly formatted. (%s)" % type(msg))
self.event.set()
def wait_until_we_have_enough_granules(self, dataset_id="", granules=4):
datastore = self.get_datastore(dataset_id)
dataset = self.dataset_management.read_dataset(dataset_id)
now = time.time()
timeout = now + 10
done = False
while not done:
if now >= timeout:
raise Timeout("Granules are not populating in time.")
if len(datastore.query_view(dataset.view_name)) >= granules:
done = True
now = time.time()
def create_dataset(self):
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
pdict = craft.create_parameters()
pdict = pdict.dump()
dataset_id = self.dataset_management.create_dataset(
"test_dataset", parameter_dict=pdict, spatial_domain=sdom, temporal_domain=tdom
)
return dataset_id
def test_coverage_ingest(self):
stream_id = self.pubsub_management.create_stream()
dataset_id = self.create_dataset()
# I freaking hate this bug
self.get_datastore(dataset_id)
ingestion_config_id = self.get_ingestion_config()
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id
)
black_box = CoverageCraft()
black_box.rdt["time"] = np.arange(20)
black_box.rdt["temp"] = np.random.random(20) * 10
black_box.sync_with_granule()
granule = black_box.to_granule()
publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id)
publisher.publish(granule)
self.wait_until_we_have_enough_granules(dataset_id, 1)
coverage = DatasetManagementService._get_coverage(dataset_id)
black_box = CoverageCraft(coverage)
black_box.sync_rdt_with_coverage()
comp = black_box.rdt["time"] == np.arange(20)
self.assertTrue(comp.all())
black_box = CoverageCraft()
black_box.rdt["time"] = np.arange(20) + 20
black_box.rdt["temp"] = np.random.random(20) * 10
black_box.sync_with_granule()
granule = black_box.to_granule()
publisher.publish(granule)
self.wait_until_we_have_enough_granules(dataset_id, 2)
coverage = DatasetManagementService._get_coverage(dataset_id)
black_box = CoverageCraft(coverage)
black_box.sync_rdt_with_coverage()
comp = black_box.rdt["time"] == np.arange(40)
self.assertTrue(comp.all())
granule = self.data_retriever.retrieve(dataset_id)
black_box = CoverageCraft()
black_box.sync_rdt_with_granule(granule)
comp = black_box.rdt["time"] == np.arange(40)
self.assertTrue(comp.all())
@attr("SMOKE")
def test_dm_end_2_end(self):
# --------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
# --------------------------------------------------------------------------------
stream_id = self.pubsub_management.create_stream()
self.launch_producer(stream_id)
# --------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
# --------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset()
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id
)
# --------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
# --------------------------------------------------------------------------------
self.wait_until_we_have_enough_granules(dataset_id, 4)
# --------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
# --------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
# --------------------------------------------------------------------------------
# Now to try the streamed approach
# --------------------------------------------------------------------------------
replay_id, stream_id = self.data_retriever.define_replay(dataset_id)
# --------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
# --------------------------------------------------------------------------------
xp = self.container.ex_manager.create_xp(self.exchange_point_name)
xn = self.container.ex_manager.create_xn_queue(self.exchange_space_name)
xn.bind("%s.data" % stream_id, xp)
subscriber = SimpleStreamSubscriber.new_subscriber(
self.container, self.exchange_space_name, self.validate_granule_subscription
)
subscriber.start()
self.data_retriever.start_replay(replay_id)
fail = False
try:
self.event.wait(10)
except gevent.Timeout:
fail = True
subscriber.stop()
self.assertTrue(not fail, "Failed to validate the data.")
def test_replay_by_time(self):
log.info("starting test...")
# --------------------------------------------------------------------------------
# Create the necessary configurations for the test
# --------------------------------------------------------------------------------
stream_id = self.pubsub_management.create_stream()
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset()
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id
)
# --------------------------------------------------------------------------------
# Create the datastore first,
# --------------------------------------------------------------------------------
# There is a race condition sometimes between the services and the process for
# the creation of the datastore and it's instance, this ensures the datastore
# exists before the process is even subscribing to data.
self.get_datastore(dataset_id)
self.publish_fake_data(stream_id)
self.wait_until_we_have_enough_granules(dataset_id, 2) # I just need two
replay_granule = self.data_retriever.retrieve(dataset_id, {"start_time": 0, "end_time": 6})
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt["time"] == np.array([0, 1, 2, 3, 4, 5])
try:
log.info("Compared granule: %s", replay_granule.__dict__)
log.info("Granule tax: %s", replay_granule.taxonomy.__dict__)
except:
pass
self.assertTrue(comp.all())
def test_last_granule(self):
# --------------------------------------------------------------------------------
# Create the necessary configurations for the test
# --------------------------------------------------------------------------------
stream_id = self.pubsub_management.create_stream()
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset()
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id
)
# --------------------------------------------------------------------------------
# Create the datastore first,
# --------------------------------------------------------------------------------
self.get_datastore(dataset_id)
self.publish_hifi(stream_id, 0)
self.publish_hifi(stream_id, 1)
self.wait_until_we_have_enough_granules(dataset_id, 2) # I just need two
replay_granule = self.data_retriever.retrieve_last_granule(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt["time"] == np.arange(10) + 10
self.assertTrue(comp.all())
def test_replay_with_parameters(self):
# --------------------------------------------------------------------------------
# Create the configurations and the dataset
# --------------------------------------------------------------------------------
stream_id = self.pubsub_management.create_stream()
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset()
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id
)
# --------------------------------------------------------------------------------
# Coerce the datastore into existence (beats race condition)
# --------------------------------------------------------------------------------
self.get_datastore(dataset_id)
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id, 4)
query = {"start_time": 0, "end_time": 20, "parameters": ["time", "temp"]}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
comp = np.arange(20) == rdt["time"]
self.assertTrue(comp.all(), "%s" % rdt.pretty_print())
self.assertEquals(set(rdt.iterkeys()), set(["time", "temp"]))
def test_repersist_data(self):
stream_id = self.pubsub_management.create_stream()
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset()
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id
)
self.get_datastore(dataset_id)
self.publish_hifi(stream_id, 0)
self.publish_hifi(stream_id, 1)
self.wait_until_we_have_enough_granules(dataset_id, 2)
self.ingestion_management.unpersist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id)
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id
)
self.publish_hifi(stream_id, 2)
self.publish_hifi(stream_id, 3)
self.wait_until_we_have_enough_granules(dataset_id, 4)
retrieved_granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(retrieved_granule)
comp = rdt["time"] == np.arange(0, 40)
self.assertTrue(comp.all(), "Uh-oh: %s" % rdt["time"])