class TestDataProcessManagementPrime(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management = DatasetManagementServiceClient()
self.resource_registry = self.container.resource_registry
self.pubsub_management = PubsubManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.validators = 0
def lc_preload(self):
config = DotDict()
config.op = 'load'
config.scenario = 'BASE,LC_TEST'
config.categories = 'ParameterFunctions,ParameterDefs,ParameterDictionary'
config.path = 'res/preload/r2_ioc'
self.container.spawn_process('preload','ion.processes.bootstrap.ion_loader','IONLoader', config)
def ctd_plain_input_data_product(self):
available_fields = [
'internal_timestamp',
'temp',
'preferred_timestamp',
'time',
'port_timestamp',
'quality_flag',
'lat',
'conductivity',
'driver_timestamp',
'lon',
'pressure']
return self.make_data_product('ctd_parsed_param_dict', 'ctd plain test', available_fields)
def ctd_plain_salinity(self):
available_fields = [
'internal_timestamp',
'preferred_timestamp',
'time',
'port_timestamp',
'quality_flag',
'lat',
'driver_timestamp',
'lon',
'salinity']
return self.make_data_product('ctd_parsed_param_dict', 'salinity', available_fields)
def ctd_plain_density(self):
available_fields = [
'internal_timestamp',
'preferred_timestamp',
'time',
'port_timestamp',
'quality_flag',
'lat',
'driver_timestamp',
'lon',
'density']
return self.make_data_product('ctd_parsed_param_dict', 'density', available_fields)
def ctd_instrument_data_product(self):
available_fields = [
'internal_timestamp',
'temp',
'preferred_timestamp',
'time',
'port_timestamp',
'quality_flag',
'lat',
'conductivity',
'driver_timestamp',
'lon',
'pressure']
return self.make_data_product('ctd_LC_TEST', 'ctd instrument', available_fields)
def make_data_product(self, pdict_name, dp_name, available_fields=[]):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(pdict_name, id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('%s stream_def' % dp_name, parameter_dictionary_id=pdict_id, available_fields=available_fields or None)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = DataProduct(name=dp_name)
dp_obj.temporal_domain = tdom
dp_obj.spatial_domain = sdom
data_product_id = self.data_product_management.create_data_product(dp_obj, stream_definition_id=stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
return data_product_id
def google_dt_data_product(self):
return self.make_data_product('google_dt', 'visual')
def ctd_derived_data_product(self):
return self.make_data_product('ctd_LC_TEST', 'ctd derived products')
def publish_to_plain_data_product(self, data_product_id):
stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
self.assertTrue(len(stream_ids))
stream_id = stream_ids.pop()
route = self.pubsub_management.read_stream_route(stream_id)
stream_definition = self.pubsub_management.read_stream_definition(stream_id=stream_id)
stream_def_id = stream_definition._id
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
now = time.time()
ntp_now = now + 2208988800 # Do not use in production, this is a loose translation
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [20.0]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['time'] = [ntp_now]
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = [None]
rdt['lat'] = [45]
rdt['conductivity'] = [4.2914]
rdt['driver_timestamp'] = [ntp_now]
rdt['lon'] = [-71]
rdt['pressure'] = [3.068]
granule = rdt.to_granule()
publisher.publish(granule)
def publish_to_data_product(self, data_product_id):
stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
self.assertTrue(len(stream_ids))
stream_id = stream_ids.pop()
route = self.pubsub_management.read_stream_route(stream_id)
stream_definition = self.pubsub_management.read_stream_definition(stream_id=stream_id)
stream_def_id = stream_definition._id
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
now = time.time()
ntp_now = now + 2208988800 # Do not use in production, this is a loose translation
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [300000]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['time'] = [ntp_now]
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = [None]
rdt['lat'] = [45]
rdt['conductivity'] = [4341400]
rdt['driver_timestamp'] = [ntp_now]
rdt['lon'] = [-71]
rdt['pressure'] = [256.8]
granule = rdt.to_granule()
publisher.publish(granule)
def setup_subscriber(self, data_product_id, callback):
stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
self.assertTrue(len(stream_ids))
stream_id = stream_ids.pop()
sub_id = self.pubsub_management.create_subscription('validator_%s'%self.validators, stream_ids=[stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
subscriber = StandaloneStreamSubscriber('validator_%s' % self.validators, callback=callback)
subscriber.start()
self.addCleanup(subscriber.stop)
self.validators+=1
return subscriber
def create_density_transform_function(self):
tf = TransformFunction(name='ctdbp_l2_density', module='ion.processes.data.transforms.ctdbp.ctdbp_L2_density', cls='CTDBP_DensityTransformAlgorithm')
tf_id = self.data_process_management.create_transform_function(tf)
self.addCleanup(self.data_process_management.delete_transform_function, tf_id)
return tf_id
def create_salinity_transform_function(self):
tf = TransformFunction(name='ctdbp_l2_salinity', module='ion.processes.data.transforms.ctdbp.ctdbp_L2_salinity', cls='CTDBP_SalinityTransformAlgorithm')
tf_id = self.data_process_management.create_transform_function(tf)
self.addCleanup(self.data_process_management.delete_transform_function, tf_id)
return tf_id
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_data_process_prime(self):
self.lc_preload()
instrument_data_product_id = self.ctd_instrument_data_product()
derived_data_product_id = self.ctd_derived_data_product()
data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[instrument_data_product_id], out_data_product_ids=[derived_data_product_id])
self.addCleanup(self.data_process_management.delete_data_process2, data_process_id)
self.data_process_management.activate_data_process2(data_process_id)
self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id)
validated = Event()
def validation(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
np.testing.assert_array_almost_equal(rdt['conductivity_L1'], np.array([42.914]))
np.testing.assert_array_almost_equal(rdt['temp_L1'], np.array([20.]))
np.testing.assert_array_almost_equal(rdt['pressure_L1'], np.array([3.068]))
np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.7144739593881]))
np.testing.assert_array_almost_equal(rdt['salinity'], np.array([30.935132729668283]))
validated.set()
self.setup_subscriber(derived_data_product_id, callback=validation)
self.publish_to_data_product(instrument_data_product_id)
self.assertTrue(validated.wait(10))
def test_older_transform(self):
input_data_product_id = self.ctd_plain_input_data_product()
conductivity_data_product_id = self.make_data_product('ctd_parsed_param_dict', 'conductivity_product', ['time', 'conductivity'])
conductivity_stream_def_id = self.get_named_stream_def('conductivity_product stream_def')
temperature_data_product_id = self.make_data_product('ctd_parsed_param_dict', 'temperature_product', ['time', 'temp'])
temperature_stream_def_id = self.get_named_stream_def('temperature_product stream_def')
pressure_data_product_id = self.make_data_product('ctd_parsed_param_dict', 'pressure_product', ['time', 'pressure'])
pressure_stream_def_id = self.get_named_stream_def('pressure_product stream_def')
dpd = DataProcessDefinition(name='ctdL0')
dpd.data_process_type = DataProcessTypeEnum.TRANSFORM
dpd.module = 'ion.processes.data.transforms.ctd.ctd_L0_all'
dpd.class_name = 'ctd_L0_all'
data_process_definition_id = self.data_process_management.create_data_process_definition(dpd)
self.addCleanup(self.data_process_management.delete_data_process_definition, data_process_definition_id)
self.data_process_management.assign_stream_definition_to_data_process_definition(conductivity_stream_def_id, data_process_definition_id, binding='conductivity')
self.data_process_management.assign_stream_definition_to_data_process_definition(temperature_stream_def_id, data_process_definition_id, binding='temperature')
self.data_process_management.assign_stream_definition_to_data_process_definition(pressure_stream_def_id, data_process_definition_id, binding='pressure')
data_process_id = self.data_process_management.create_data_process2(data_process_definition_id=data_process_definition_id, in_data_product_ids=[input_data_product_id], out_data_product_ids=[conductivity_data_product_id, temperature_data_product_id, pressure_data_product_id])
self.addCleanup(self.data_process_management.delete_data_process2, data_process_id)
self.data_process_management.activate_data_process2(data_process_id)
self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id)
conductivity_validated = Event()
def validate_conductivity(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
np.testing.assert_array_almost_equal(rdt['conductivity'], np.array([4.2914]))
conductivity_validated.set()
self.setup_subscriber(conductivity_data_product_id, callback=validate_conductivity)
temperature_validated = Event()
def validate_temperature(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
np.testing.assert_array_almost_equal(rdt['temp'], np.array([20.0]))
temperature_validated.set()
self.setup_subscriber(temperature_data_product_id, callback=validate_temperature)
pressure_validated = Event()
def validate_pressure(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
np.testing.assert_array_almost_equal(rdt['pressure'], np.array([3.068]))
pressure_validated.set()
self.setup_subscriber(pressure_data_product_id, callback=validate_pressure)
self.publish_to_plain_data_product(input_data_product_id)
self.assertTrue(conductivity_validated.wait(10))
self.assertTrue(temperature_validated.wait(10))
self.assertTrue(pressure_validated.wait(10))
def get_named_stream_def(self, name):
stream_def_ids, _ = self.resource_registry.find_resources(name=name, restype=RT.StreamDefinition, id_only=True)
return stream_def_ids[0]
def test_actors(self):
input_data_product_id = self.ctd_plain_input_data_product()
output_data_product_id = self.ctd_plain_density()
actor = self.create_density_transform_function()
route = {input_data_product_id: {output_data_product_id: actor}}
config = DotDict()
config.process.routes = route
config.process.params.lat = 45.
config.process.params.lon = -71.
data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[input_data_product_id], out_data_product_ids=[output_data_product_id], configuration=config)
self.addCleanup(self.data_process_management.delete_data_process2, data_process_id)
self.data_process_management.activate_data_process2(data_process_id)
self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id)
validated = Event()
def validation(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
# The value I use is a double, the value coming back is only a float32 so there's some data loss but it should be precise to the 4th digit
np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.6839775385847]), decimal=4)
validated.set()
self.setup_subscriber(output_data_product_id, callback=validation)
self.publish_to_plain_data_product(input_data_product_id)
self.assertTrue(validated.wait(10))
def test_multi_in_out(self):
input1 = self.ctd_plain_input_data_product()
input2 = self.make_data_product('ctd_parsed_param_dict', 'input2')
density_dp_id = self.ctd_plain_density()
salinity_dp_id = self.ctd_plain_salinity()
density_actor = self.create_density_transform_function()
salinity_actor = self.create_salinity_transform_function()
routes = {
input1 : {
density_dp_id : density_actor,
salinity_dp_id : salinity_actor
},
input2 : {
density_dp_id : density_actor
}
}
config = DotDict()
config.process.routes = routes
config.process.params.lat = 45.
config.process.params.lon = -71.
data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[input1, input2], out_data_product_ids=[density_dp_id, salinity_dp_id], configuration=config)
self.addCleanup(self.data_process_management.delete_data_process2, data_process_id)
self.data_process_management.activate_data_process2(data_process_id)
self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id)
density_validated = Event()
salinity_validated = Event()
def density_validation(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.6839775385847]), decimal=4)
density_validated.set()
def salinity_validation(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
np.testing.assert_array_almost_equal(rdt['salinity'], np.array([30.93513240786831]), decimal=4)
salinity_validated.set()
self.setup_subscriber(density_dp_id, callback=density_validation)
self.setup_subscriber(salinity_dp_id, callback=salinity_validation)
self.publish_to_plain_data_product(input1)
self.assertTrue(density_validated.wait(10))
self.assertTrue(salinity_validated.wait(10))
density_validated.clear()
salinity_validated.clear()
self.publish_to_plain_data_product(input2)
self.assertTrue(density_validated.wait(10))
self.assertFalse(salinity_validated.wait(0.75))
density_validated.clear()
salinity_validated.clear()
def test_visual_transform(self):
input_data_product_id = self.ctd_plain_input_data_product()
output_data_product_id = self.google_dt_data_product()
dpd = DataProcessDefinition(name='visual transform')
dpd.data_process_type = DataProcessTypeEnum.TRANSFORM
dpd.module = 'ion.processes.data.transforms.viz.google_dt'
dpd.class_name = 'VizTransformGoogleDT'
#--------------------------------------------------------------------------------
# Walk before we base jump
#--------------------------------------------------------------------------------
data_process_definition_id = self.data_process_management.create_data_process_definition(dpd)
self.addCleanup(self.data_process_management.delete_data_process_definition, data_process_definition_id)
data_process_id = self.data_process_management.create_data_process2(data_process_definition_id=data_process_definition_id, in_data_product_ids=[input_data_product_id], out_data_product_ids=[output_data_product_id])
self.addCleanup(self.data_process_management.delete_data_process2,data_process_id)
self.data_process_management.activate_data_process2(data_process_id)
self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id)
validated = Event()
def validation(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
self.assertTrue(rdt['google_dt_components'] is not None)
validated.set()
self.setup_subscriber(output_data_product_id, callback=validation)
self.publish_to_plain_data_product(input_data_product_id)
self.assertTrue(validated.wait(10))
