def test_tmpsf_arrays(self):
self.preload_tmpsf()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('tmpsf_sample', id_only=True)
stream_def_id = self.create_stream_definition('tmpsf', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('tmpsf', stream_def_id=stream_def_id)
self.activate_data_product(data_product_id)
rdt = ParameterHelper.rdt_for_data_product(data_product_id)
tomato = {'quality_flag': 'ok', 'preferred_timestamp':
'port_timestamp', 'internal_timestamp': 3223662780.0,
'stream_name': 'tmpsf_sample', 'values': [{'value_id':
'timestamp', 'value': 3223662780.0}, {'value_id':
'temperature', 'value': [21.4548, 21.0132, 20.9255,
21.1266, 21.1341, 21.5606, 21.2156, 21.4749,
21.3044, 21.132, 21.1798, 21.2352, 21.3488,
21.1214, 21.6426, 21.1479, 21.0069, 21.5426,
21.3204, 21.2402, 21.3968, 21.4371, 21.0411,
21.4361]}, {'value_id': 'battery_voltage', 'value':
11.5916}, {'value_id': 'serial_number',
'value': '021964'}], 'port_timestamp':
1378230448.439269, 'driver_timestamp':
3587219248.444593, 'pkt_format_id':
'JSON_Data', 'pkt_version': 1}
from ion.agents.populate_rdt import populate_rdt
rdt = populate_rdt(rdt, [tomato])
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
breakpoint(locals())
def test_example_preload(self):
print 'preloading...'
self.preload_example1()
data_product_ids, _ = self.container.resource_registry.find_resources_ext(alt_id='DPROD102', alt_id_ns='PRE')
data_product_id = data_product_ids[0]
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
with DirectCoverageAccess() as dca:
dca.upload_calibration_coefficients(dataset_id, 'test_data/sbe16coeffs.csv', 'test_data/sbe16coeffs.yml')
ph = ParameterHelper(self.dataset_management, self.addCleanup)
rdt = ph.rdt_for_data_product(data_product_id)
rdt['time'] = [time.time() + 2208988800]
rdt['temperature'] = [248471]
rdt['pressure'] = [528418]
rdt['conductivity'] = [1673175]
rdt['thermistor_temperature']=[24303]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ph.publish_rdt_to_data_product(data_product_id, rdt)
dataset_monitor.event.wait(10)
g = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(g)
breakpoint(locals())
def test_serialize_compatability(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('ctd extended', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('ctd1', 'xp1', stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
sub_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
verified = Event()
def verifier(msg, route, stream_id):
for k,v in msg.record_dictionary.iteritems():
if v is not None:
self.assertIsInstance(v, np.ndarray)
rdt = RecordDictionaryTool.load_from_granule(msg)
for k,v in rdt.iteritems():
self.assertIsInstance(rdt[k], np.ndarray)
self.assertIsInstance(v, np.ndarray)
verified.set()
subscriber = StandaloneStreamSubscriber('sub1', callback=verifier)
subscriber.start()
self.addCleanup(subscriber.stop)
publisher = StandaloneStreamPublisher(stream_id,route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
ph.fill_rdt(rdt,10)
publisher.publish(rdt.to_granule())
self.assertTrue(verified.wait(60))
def test_example2_preload(self):
print 'preloading...'
self.preload_example2()
data_product_ids, _ = self.container.resource_registry.find_resources_ext(alt_id='DPROD104', alt_id_ns='PRE')
data_product_id = data_product_ids[0]
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
with DirectCoverageAccess() as dca:
dca.upload_calibration_coefficients(dataset_id, 'test_data/vel3d_coeff.csv', 'test_data/vel3d_coeff.yml')
from ion_functions.data.test.test_vel_functions import TS, VE, VN, VU
rdt = ParameterHelper.rdt_for_data_product(data_product_id)
rdt['time'] = [time.time() + 2208988800]
rdt['velocity_east'] = [VE[0]]
rdt['velocity_north'] = [VN[0]]
rdt['velocity_up'] = [VU[0]]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
dataset_monitor.event.wait(10)
g = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(g)
breakpoint(locals())
def check_tempsf_instrument_data_product(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator, info_list)
if not passing: return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['temperature'] = [[ 25.3884, 26.9384, 24.3394, 23.3401, 22.9832,
29.4434, 26.9873, 15.2883, 16.3374, 14.5883, 15.7253, 18.4383,
15.3488, 17.2993, 10.2111, 11.5993, 10.9345, 9.4444, 9.9876,
10.9834, 11.0098, 5.3456, 4.2994, 4.3009]]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.event.wait(20))
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['temperature'], [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009]])
return passing
def test_qc_events(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_qc_pdict()
stream_def_id = self.pubsub_management.create_stream_definition('qc stream def', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('qc stream', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
config = DotDict()
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id, config=config)
self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id)
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = np.arange(10) * 3
verified = Event()
def verification(event, *args, **kwargs):
self.assertEquals(event.qc_parameter, 'temp_qc')
self.assertEquals(event.temporal_value, 7)
verified.set()
es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=dataset_id, callback=verification, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(verified.wait(10))
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process = TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(
node=self.container.node, process=process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client,
self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
class TestDynamicParameters(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = self.ph.create_simple_cc_pdict()
self.stream_def_id = self.pubsub_management.create_stream_definition('Calibration Coefficients', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, self.stream_def_id)
def test_coefficient_compatibility(self):
data_product_id = self.create_data_product(name='Calibration Coefficient Test Data product', stream_def_id=self.stream_def_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = [10] * 10
rdt['cc_coefficient'] = [2] * 10
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.wait())
rdt2 = RecordDictionaryTool.load_from_granule(self.data_retriever.retrieve(dataset_id))
np.testing.assert_array_equal(rdt2['offset'],[12]*10)
def test_qc_interval_integration(self):
# 1 need to make a dataset that only has one discrete qc violation
# 2 Launch the process
# 3 Setup the scheduler to run it say three times
# 4 Get the Events and verify the data
#--------------------------------------------------------------------------------
# Make a dataset that has only one discrete qc violation
#--------------------------------------------------------------------------------
dp_id, dataset_id, stream_def_id = self.make_data_product()
ph = ParameterHelper(self.dataset_management, self.addCleanup)
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
for rdt in self.populate_vectors(stream_def_id, 1, lambda x : [41] + [39] * (x-1)):
ph.publish_rdt_to_data_product(dp_id, rdt)
self.assertTrue(monitor.event.wait(10)) # Give it 10 seconds to populate
#--------------------------------------------------------------------------------
# Launch the process
#--------------------------------------------------------------------------------
interval_key = uuid4().hex
config = DotDict()
config.process.interval_key = interval_key
config.process.qc_params = ['glblrng_qc'] # The others are tested in other tests for completeness
self.sync_launch(config)
async_queue = Queue()
def callback(event, *args, **kwargs):
times = event.temporal_values
self.assertEquals(len(times), 1)
async_queue.put(1)
es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=dp_id, callback=callback, auto_delete=True)
es.start()
self.addCleanup(es.stop)
#--------------------------------------------------------------------------------
# Setup the scheduler
#--------------------------------------------------------------------------------
timer_id = self.scheduler_service.create_interval_timer(start_time=time.time(),
end_time=time.time()+13,
interval=5,
event_origin=interval_key)
#--------------------------------------------------------------------------------
# Get the events and verify them
#--------------------------------------------------------------------------------
try:
for i in xrange(2):
async_queue.get(timeout=10)
except Empty:
raise AssertionError('QC Events not raised')
def create_lookup_rdt(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsub_management.create_stream_definition('lookup', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
return rdt
def create_lookup_rdt(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsub_management.create_stream_definition('lookup', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':"GA03FLMA-RI001-13-CTDMOG999"})
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
return rdt
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = self.ph.create_simple_cc_pdict()
self.stream_def_id = self.pubsub_management.create_stream_definition(
'Calibration Coefficients', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
self.stream_def_id)
def test_hydrophone_simulator(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.crete_simple_array_pdict()
stream_def_id = self.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('ctd hydrophone', stream_def_id=stream_def_id)
self.activate_data_product(data_product_id)
s = HydrophoneSimulator(data_product_id, interval=4)
breakpoint(locals())
s.stop()
def make_data_product(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_simple_qc_pdict()
stream_def_id = self.create_stream_definition('global range', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.populate_qc_tables()
dp_id = self.create_data_product('qc data product', stream_def_id=stream_def_id)
self.activate_data_product(dp_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(dp_id)
return dp_id, dataset_id, stream_def_id
def make_data_product(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_simple_qc_pdict()
stream_def_id = self.create_stream_definition(
'global range',
parameter_dictionary_id=pdict_id,
stream_configuration={'reference_designator': 'QCTEST'})
self.populate_qc_tables()
dp_id = self.create_data_product('qc data product',
stream_def_id=stream_def_id)
self.activate_data_product(dp_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(
dp_id)
return dp_id, dataset_id, stream_def_id
def run(self):
while not self.finished.wait(self.interval):
rdt = ParameterHelper.rdt_for_data_product(self.data_product_id)
now = time.time()
if self.simple_time:
rdt['time'] = [self.i]
else:
rdt['time'] = np.array([now + 2208988800])
rdt['temp'] = self.float_range(10,14,np.array([now]))
rdt['pressure'] = self.float_range(11,12,np.array([now]))
rdt['lat'] = [41.205]
rdt['lon'] = [-71.74]
rdt['conductivity'] = self.float_range(3.3,3.5,np.array([now]))
rdt['driver_timestamp'] = np.array([now + 2208988800])
rdt['preferred_timestamp'] = ['driver_timestamp']
ParameterHelper.publish_rdt_to_data_product(self.data_product_id, rdt)
self.i += 1
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = self.ph.create_simple_cc_pdict()
self.stream_def_id = self.pubsub_management.create_stream_definition('Calibration Coefficients', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, self.stream_def_id)
def test_create_dataset(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
# Yield to other greenlets, had an issue with connectivity
gevent.sleep(1)
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
breakpoint(locals(), globals())
def setup_resource(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
# Yield to other greenlets, had an issue with connectivity
gevent.sleep(1)
self.offering_id = dataset_id
def test_fdt_created_during(self):
# generate a data product and check that the FDT exists
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
print coverage_path
print "--------------------------------"
def test_get_data_from_FDW(self):
# generate a data product and check that the FDW can get data
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
print coverage_path
print "--------------------------------"
#verify table exists in the DB (similar to above)
# ....code...
# check that the geoserver layer exists as above
# ... code ....
# make a WMS/WFS request...somet like this (or both)
url = self.gs_host + '/geoserver/geonode/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=geonode:ooi_' + dataset_id + '_ooi&maxFeatures=1&outputFormat=csv'
r = requests.get(url)
assertTrue(r.status_code == 200)
def check_presta_instrument_data_products(self, reference_designator):
# Check the parsed data product make sure it's got everything it needs and can be published persisted etc.
# Absolute Pressure (SFLPRES_L0) is what comes off the instrumnet, SFLPRES_L1 is a pfunc
# Let's go ahead and publish some fake data!!!
# According to https://alfresco.oceanobservatories.org/alfresco/d/d/workspace/SpacesStore/63e16865-9d9e-4b11-b0b3-d5658faa5080/1341-00230_Data_Product_Spec_SFLPRES_OOI.pdf
# Appendix A. Example 1.
# p_psia_tide = 14.8670
# the tide should be 10.2504
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800.
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['absolute_pressure'] = [14.8670]
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'],
[10.2504], 4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(
dataset_monitor.wait()) # Bumped to 20 to keep buildbot happy
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'],
[10.2504], 4)
passing &= self.assert_array_almost_equal(rdt['absolute_pressure'],
[14.8670], 4)
return passing
def test_pydap(self):
if not CFG.get_safe('bootstrap.use_pydap',False):
raise unittest.SkipTest('PyDAP is off (bootstrap.use_pydap)')
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt,10)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity
pydap_host = CFG.get_safe('server.pydap.host','localhost')
pydap_port = CFG.get_safe('server.pydap.port',8001)
url = 'http://%s:%s/%s' %(pydap_host, pydap_port, dataset_id)
ds = open_url(url)
np.testing.assert_array_equal(ds['time'][:], np.arange(10))
untested = []
for k,v in rdt.iteritems():
if k==rdt.temporal_parameter:
continue
context = rdt.context(k)
if isinstance(context.param_type, QuantityType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, ArrayType):
values = np.empty(rdt[k].shape, dtype='O')
for i,obj in enumerate(rdt[k]):
values[i] = str(obj)
np.testing.assert_array_equal(ds[k][k][:][0], values)
elif isinstance(context.param_type, ConstantType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, CategoryType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
else:
untested.append('%s (%s)' % (k,context.param_type))
if untested:
raise AssertionError('Untested parameters: %s' % untested)
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = self.ph.create_simple_qc_pdict()
self.stream_def_id = self.pubsub_management.create_stream_definition('global range', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.addCleanup(self.pubsub_management.delete_stream_definition, self.stream_def_id)
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.svm = StoredValueManager(self.container)
def test_create_dataset_verify_geoserver_layer(self):
#generate layer and check that the service created it in geoserver
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
# verify that the layer exists in geoserver
try:
r = requests.get(self.gs_rest_url + '/layers/ooi_' + dataset_id +
'_ooi.xml',
auth=(self.username, self.PASSWORD))
self.assertTrue(r.status_code == 200)
except Exception as e:
log.error("check service and layer exist...%s", e)
self.assertTrue(False)
def make_large_dataset(self, temp_vector):
monitor_queue = Queue()
# Make 27 hours of data
ph = ParameterHelper(self.dataset_management, self.addCleanup)
data_product_id, dataset_id, stream_def_id = self.make_data_product()
es = EventSubscriber(event_type=OT.DatasetModified, origin=dataset_id, auto_delete=True, callback = lambda *args, **kwargs : monitor_queue.put(1))
es.start()
self.addCleanup(es.stop)
for rdt in self.populate_vectors(stream_def_id, 3, temp_vector):
ph.publish_rdt_to_data_product(data_product_id, rdt)
try:
for i in xrange(3):
monitor_queue.get(timeout=10)
except Empty:
raise AssertionError('Failed to populate dataset in time')
return data_product_id
def test_global_range_lookup(self):
reference_designator = "CE01ISSM-MF005-01-CTDBPC999"
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_simple_qc_pdict()
svm = StoredValueManager(self.container)
doc_key = 'grt_%s_TEMPWAT' % reference_designator
svm.stored_value_cas(doc_key, {'grt_min_value':-2, 'grt_max_value':40})
stream_def_id = self.pubsub_management.create_stream_definition('qc parsed', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':reference_designator})
self.addCleanup(self.pubsub_management.delete_stream_definition,stream_def_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20]
rdt.fetch_lookup_values()
min_field = [i for i in rdt.fields if 'grt_min_value' in i][0]
max_field = [i for i in rdt.fields if 'grt_max_value' in i][0]
np.testing.assert_array_almost_equal(rdt[min_field], [-2.])
np.testing.assert_array_almost_equal(rdt[max_field], [40.])
np.testing.assert_array_almost_equal(rdt['tempwat_glblrng_qc'],[1])
def test_lctest_preload(self):
self.preload_lctest()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('sparse_dict', id_only=True)
stream_def_id = self.create_stream_definition('sparse_example', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('sparse_example', stream_def_id=stream_def_id)
self.activate_data_product(data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
rdt = ParameterHelper.rdt_for_data_product(data_product_id)
rdt['time'] = [time.time() + 2208988800]
rdt['sparse_float'] = [3.14159265358979323]
rdt['sparse_double'] = [2.7182818284590452353602874713526624977572470936999595]
rdt['sparse_int'] = [131071] # 6th mersenne prime
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
dataset_monitor.event.wait(10)
for i in xrange(10):
dataset_monitor.event.clear()
rdt = ParameterHelper.rdt_for_data_product(data_product_id)
rdt['time'] = [time.time() + 2208988800]
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
dataset_monitor.event.wait(10)
g = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(g)
breakpoint(locals())
def check_presta_instrument_data_products(self, reference_designator):
# Check the parsed data product make sure it's got everything it needs and can be published persisted etc.
# Absolute Pressure (SFLPRES_L0) is what comes off the instrumnet, SFLPRES_L1 is a pfunc
# Let's go ahead and publish some fake data!!!
# According to https://alfresco.oceanobservatories.org/alfresco/d/d/workspace/SpacesStore/63e16865-9d9e-4b11-b0b3-d5658faa5080/1341-00230_Data_Product_Spec_SFLPRES_OOI.pdf
# Appendix A. Example 1.
# p_psia_tide = 14.8670
# the tide should be 10.2504
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator, info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800.
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['absolute_pressure'] = [14.8670]
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.event.wait(20)) # Bumped to 20 to keep buildbot happy
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4)
passing &= self.assert_array_almost_equal(rdt['absolute_pressure'], [14.8670], 4)
return passing
def run(self):
connection = uuid4().hex
while not self.finished.wait(self.interval):
rdt = ParameterHelper.rdt_for_data_product(self.data_product_id)
now = time.time()
if self.simple_time:
rdt["time"] = [self.i]
else:
rdt["time"] = np.array([now + 2208988800])
rdt["temp"] = self.float_range(10, 14, np.array([now]))
rdt["pressure"] = self.float_range(11, 12, np.array([now]))
rdt["lat"] = [41.205]
rdt["lon"] = [-71.74]
rdt["conductivity"] = self.float_range(3.3, 3.5, np.array([now]))
rdt["driver_timestamp"] = np.array([now + 2208988800])
rdt["preferred_timestamp"] = ["driver_timestamp"]
if self.connection:
ParameterHelper.publish_rdt_to_data_product(
self.data_product_id, rdt, connection_id=connection, connection_index=self.i
)
else:
ParameterHelper.publish_rdt_to_data_product(self.data_product_id, rdt)
self.i += 1
def make_large_dataset(self, temp_vector):
monitor_queue = Queue()
# Make 27 hours of data
ph = ParameterHelper(self.dataset_management, self.addCleanup)
data_product_id, dataset_id, stream_def_id = self.make_data_product()
es = EventSubscriber(
event_type=OT.DatasetModified,
origin=dataset_id,
auto_delete=True,
callback=lambda *args, **kwargs: monitor_queue.put(1))
es.start()
self.addCleanup(es.stop)
for rdt in self.populate_vectors(stream_def_id, 3, temp_vector):
ph.publish_rdt_to_data_product(data_product_id, rdt)
try:
for i in xrange(3):
monitor_queue.get(timeout=10)
except Empty:
raise AssertionError('Failed to populate dataset in time')
return data_product_id
class TestDynamicParameters(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = self.ph.create_simple_cc_pdict()
self.stream_def_id = self.pubsub_management.create_stream_definition(
'Calibration Coefficients', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
self.stream_def_id)
def test_coefficient_compatibility(self):
data_product_id = self.create_data_product(
name='Calibration Coefficient Test Data product',
stream_def_id=self.stream_def_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = [10] * 10
rdt['cc_coefficient'] = [2] * 10
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(
data_product_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.wait())
rdt2 = RecordDictionaryTool.load_from_granule(
self.data_retriever.retrieve(dataset_id))
np.testing.assert_array_equal(rdt2['offset'], [12] * 10)
def test_coverage_transform(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id)
self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id)
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = ph.get_rdt(stream_def_id)
ph.fill_parsed_rdt(rdt)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], rdt['time'])
np.testing.assert_array_almost_equal(rdt_out['temp'], rdt['temp'])
np.testing.assert_allclose(rdt_out['conductivity_L1'], np.array([42.914]))
np.testing.assert_allclose(rdt_out['temp_L1'], np.array([20.]))
np.testing.assert_allclose(rdt_out['pressure_L1'], np.array([3.068]))
np.testing.assert_allclose(rdt_out['density'], np.array([1021.7144739593881], dtype='float32'))
np.testing.assert_allclose(rdt_out['salinity'], np.array([30.935132729668283], dtype='float32'))
def check_tempsf_instrument_data_product(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing: return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['temperature'] = [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009
]]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['temperature'], [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009
]])
return passing
def test_create_dataset(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
# Yield to other greenlets, had an issue with connectivity
gevent.sleep(1)
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
breakpoint(locals(), globals())
def test_fdt_created_during(self):
# generate a data product and check that the FDT exists
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
print coverage_path
print "--------------------------------"
def test_get_data_from_FDW(self):
# generate a data product and check that the FDW can get data
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
print coverage_path
print "--------------------------------"
#verify table exists in the DB (similar to above)
# ....code...
# check that the geoserver layer exists as above
# ... code ....
# make a WMS/WFS request...somet like this (or both)
url = self.gs_host+'/geoserver/geonode/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=geonode:ooi_' + dataset_id + '_ooi&maxFeatures=1&outputFormat=csv'
r = requests.get(url)
assertTrue(r.status_code == 200)
def test_coverage_transform(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
stream_id, route = self.pubsub_management.create_stream(
'example',
exchange_point=self.exchange_point_name,
stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(
stream_id=stream_id,
ingestion_configuration_id=ingestion_config_id,
dataset_id=dataset_id)
self.addCleanup(self.ingestion_management.unpersist_data_stream,
stream_id, ingestion_config_id)
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = ph.get_rdt(stream_def_id)
ph.fill_parsed_rdt(rdt)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], rdt['time'])
np.testing.assert_array_almost_equal(rdt_out['temp'], rdt['temp'])
np.testing.assert_array_almost_equal(rdt_out['conductivity_L1'],
np.array([42.914]))
np.testing.assert_array_almost_equal(rdt_out['temp_L1'],
np.array([20.]))
np.testing.assert_array_almost_equal(rdt_out['pressure_L1'],
np.array([3.068]))
np.testing.assert_array_almost_equal(
rdt_out['density'], np.array([1021.7144739593881],
dtype='float32'))
np.testing.assert_array_almost_equal(
rdt_out['salinity'], np.array([30.935132729668283],
dtype='float32'))
def test_create_dataset_verify_geoserver_layer(self):
#generate layer and check that the service created it in geoserver
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
# verify that the layer exists in geoserver
try:
r = requests.get(self.gs_rest_url + '/layers/ooi_' + dataset_id + '_ooi.xml', auth=(self.username, self.PASSWORD))
self.assertTrue(r.status_code == 200)
except Exception as e:
log.error("check service and layer exist...%s", e)
self.assertTrue(False)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process=TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process = process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client, self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = self.container.resource_registry
self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
self.data_acquisition_management = DataAcquisitionManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.instrument_management = InstrumentManagementServiceClient()
self.discovery = DiscoveryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.user_notification = UserNotificationServiceClient()
self.observatory_management = ObservatoryManagementServiceClient()
self.visualization = VisualizationServiceClient()
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
self.ctd_count = 0
def setup_resource(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
# Yield to other greenlets, had an issue with connectivity
gevent.sleep(1)
self.offering_id = dataset_id
class TestQCFunctions(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = self.ph.create_simple_qc_pdict()
self.stream_def_id = self.pubsub_management.create_stream_definition('global range', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.addCleanup(self.pubsub_management.delete_stream_definition, self.stream_def_id)
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.svm = StoredValueManager(self.container)
def test_global_range_test(self):
self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def test_spike_test(self): # I know how redundant this sounds
self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1])
def test_stuck_value_test(self):
self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
class TestTransformWorker(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process = TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(
node=self.container.node, process=process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client,
self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def push_granule(self, data_product_id):
'''
Publishes and monitors that the granule arrived
'''
datasets, _ = self.rrclient.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
dataset_monitor = DatasetMonitor(datasets[0])
rdt = self.ph.rdt_for_data_product(data_product_id)
self.ph.fill_parsed_rdt(rdt)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
assert dataset_monitor.wait()
dataset_monitor.stop()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.dp_list = []
self.data_process_objs = []
self._output_stream_ids = []
self.granule_verified = Event()
self.worker_assigned_event_verified = Event()
self.dp_created_event_verified = Event()
self.heartbeat_event_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
self.start_event_listener()
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.dp_list.append(dataprocess_id)
# validate the repository for data product algorithms persists the new resources NEW SA-1
# create_data_process call created one of each
dpd_ids, _ = self.rrclient.find_resources(
restype=OT.DataProcessDefinition, id_only=False)
# there will be more than one becuase of the DPDs that reperesent the PFs in the data product above
self.assertTrue(dpd_ids is not None)
dp_ids, _ = self.rrclient.find_resources(restype=OT.DataProcess,
id_only=False)
# only one DP becuase the PFs that are in the code dataproduct above are not activated yet.
self.assertEquals(len(dp_ids), 1)
# validate the name and version label NEW SA - 2
dataprocessdef_obj = self.dataprocessclient.read_data_process_definition(
dataprocessdef_id)
self.assertEqual(dataprocessdef_obj.version_label, '1.0a')
self.assertEqual(dataprocessdef_obj.name, 'add_arrays')
# validate that the DPD has an attachment NEW SA - 21
attachment_ids, assoc_ids = self.rrclient.find_objects(
dataprocessdef_id, PRED.hasAttachment, RT.Attachment, True)
self.assertEqual(len(attachment_ids), 1)
attachment_obj = self.rrclient.read_attachment(attachment_ids[0])
log.debug('attachment: %s', attachment_obj)
# validate that the data process resource has input and output data products associated
# L4-CI-SA-RQ-364 and NEW SA-3
outproduct_ids, assoc_ids = self.rrclient.find_objects(
dataprocess_id, PRED.hasOutputProduct, RT.DataProduct, True)
self.assertEqual(len(outproduct_ids), 1)
inproduct_ids, assoc_ids = self.rrclient.find_objects(
dataprocess_id, PRED.hasInputProduct, RT.DataProduct, True)
self.assertEqual(len(inproduct_ids), 1)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 2)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(output_data_product_provenance[
output_data_product_id[0]]['parents'][self.input_dp_id]
['data_process_definition_id'] == dataprocessdef_id)
# NEW SA - 4 | Data processing shall include the appropriate data product algorithm name and version number in
# the metadata of each output data product created by the data product algorithm.
output_data_product_obj, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=False)
self.assertTrue(output_data_product_obj[0].name != None)
self.assertTrue(output_data_product_obj[0]._rev != None)
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=dataprocess_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id,
stream_route=stream_route)
for n in range(1, 101):
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
# validate that the output granule is received and the updated value is correct
self.assertTrue(self.granule_verified.wait(self.wait_time))
# validate that the data process loaded into worker event is received (L4-CI-SA-RQ-182)
self.assertTrue(
self.worker_assigned_event_verified.wait(self.wait_time))
# validate that the data process create (with data product ids) event is received (NEW SA -42)
self.assertTrue(self.dp_created_event_verified.wait(self.wait_time))
# validate that the data process heartbeat event is received (for every hundred granules processed) (L4-CI-SA-RQ-182)
#this takes a while so set wait limit to large value
self.assertTrue(self.heartbeat_event_verified.wait(200))
# validate that the code from the transform function can be retrieve via inspect_data_process_definition
src = self.dataprocessclient.inspect_data_process_definition(
dataprocessdef_id)
self.assertIn('def add_arrays(a, b)', src)
# now delete the DPD and DP then verify that the resources are retired so that information required for provenance are still available
self.dataprocessclient.delete_data_process(dataprocess_id)
self.dataprocessclient.delete_data_process_definition(
dataprocessdef_id)
in_dp_objs, _ = self.rrclient.find_objects(
subject=dataprocess_id,
predicate=PRED.hasInputProduct,
object_type=RT.DataProduct,
id_only=True)
self.assertTrue(in_dp_objs is not None)
dpd_objs, _ = self.rrclient.find_subjects(
subject_type=RT.DataProcessDefinition,
predicate=PRED.hasDataProcess,
object=dataprocess_id,
id_only=True)
self.assertTrue(dpd_objs is not None)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_instrumentdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
instDevice_obj, _ = self.rrclient.find_resources(
restype=RT.InstrumentDevice, name='test_ctd_device')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice,
name='test_ctd_device',
description="test_ctd_device",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.addCleanup(self.dataprocessclient.delete_data_process,
dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition,
dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(instDevice_id in output_data_product_provenance[
self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[instDevice_id]['type']
== 'InstrumentDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_platformdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
platform_device_obj, _ = self.rrclient.find_resources(
restype=RT.PlatformDevice, name='TestPlatform')
if platform_device_obj:
platform_device_id = platform_device_obj[0]._id
else:
platform_device_obj = IonObject(RT.PlatformDevice,
name='TestPlatform',
description="TestPlatform",
serial_number="12345")
platform_device_id = self.imsclient.create_platform_device(
platform_device=platform_device_obj)
self.damsclient.assign_data_product(
input_resource_id=platform_device_id,
data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.addCleanup(self.dataprocessclient.delete_data_process,
dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition,
dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(platform_device_id in output_data_product_provenance[
self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[platform_device_id]
['type'] == 'PlatformDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_event_transform_worker(self):
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# test that a data process (type: data-product-in / event-out) can be defined and launched.
# verify that event fields are correctly populated
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
# create the DPD and two DPs
self.event_data_process_id = self.create_event_data_processes()
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=self.event_data_process_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_event_transform_worker subscription_obj: %s',
subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id,
stream_route=stream_route)
self.start_event_transform_listener()
self.data_modified = Event()
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
self.assertTrue(self.event_verified.wait(self.wait_time))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_bad_argument_map(self):
self._output_stream_ids = []
# test that a data process (type: data-product-in / data-product-out) parameter mapping it validated during
# data process creation and that the correct exception is raised for both input and output.
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
# Set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
# create the data process with invalid argument map
argument_map = {"arr1": "foo", "arr2": "bar"}
output_param = "salinity"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(
ex.message,
"Input data product does not contain the parameters defined in argument map"
)
# create the data process with invalid output parameter name
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "foo"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(
ex.message,
"Output data product does not contain the output parameter name provided"
)
def create_event_data_processes(self):
# two data processes using one transform and one DPD
argument_map = {"a": "salinity"}
# set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='validate_salinity_array',
description='validate_salinity_array',
function='validate_salinity_array',
module="ion.processes.data.transforms.test.test_transform_worker",
arguments=['a'],
function_type=TransformFunctionType.TRANSFORM)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='validate_salinity_array',
description='validate_salinity_array',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id,
add_array_dpd_id,
binding='validate_salinity_array')
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=None,
argument_map=argument_map)
self.damsclient.register_process(dp1_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
dp1_data_process_id)
return dp1_data_process_id
def create_data_process(self):
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "salinity"
# set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
version_label='1.0a')
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp1_data_process_id)
#self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id)
# add an attachment object to this DPD to test new SA-21
import msgpack
attachment_content = 'foo bar'
attachment_obj = IonObject(RT.Attachment,
name='test_attachment',
attachment_type=AttachmentType.ASCII,
content_type='text/plain',
content=msgpack.packb(attachment_content))
att_id = self.rrclient.create_attachment(add_array_dpd_id,
attachment_obj)
self.addCleanup(self.rrclient.delete_attachment, att_id)
return add_array_dpd_id, dp1_data_process_id, dp1_func_output_dp_id
def create_output_data_product(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject(RT.DataProduct,
name='data_process1_data_product',
description='output of add array func')
dp1_func_output_dp_id = self.dataproductclient.create_data_product(
dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp1_func_output_dp_id)
# retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_ids.append(stream_ids[0])
subscription_id = self.pubsub_client.create_subscription(
'validator', data_product_ids=[dp1_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription,
subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s',
stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
self.granule_verified.set()
validator = StandaloneStreamSubscriber('validator',
callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
subscription_id)
return dp1_func_output_dp_id
def validate_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
data_process_event = args[0]
log.debug("DataProcessStatusEvent: %s",
str(data_process_event.__dict__))
# if data process already created, check origin
if self.dp_list:
self.assertIn(data_process_event.origin, self.dp_list)
# if this is a heartbeat event then 100 granules have been processed
if 'data process status update.' in data_process_event.description:
self.heartbeat_event_verified.set()
else:
# else check that this is the assign event
if 'Data process assigned to transform worker' in data_process_event.description:
self.worker_assigned_event_verified.set()
elif 'Data process created for data product' in data_process_event.description:
self.dp_created_event_verified.set()
def validate_output_granule(self, msg, route, stream_id):
self.assertIn(stream_id, self._output_stream_ids)
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule rdt: %s', rdt)
sal_val = rdt['salinity']
np.testing.assert_array_equal(sal_val, np.array([3]))
def start_event_listener(self):
es = EventSubscriber(event_type=OT.DataProcessStatusEvent,
callback=self.validate_event)
es.start()
self.addCleanup(es.stop)
def validate_transform_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
status_alert_event = args[0]
np.testing.assert_array_equal(status_alert_event.origin,
self.stream_id)
np.testing.assert_array_equal(status_alert_event.values,
np.array([self.event_data_process_id]))
log.debug("DeviceStatusAlertEvent: %s",
str(status_alert_event.__dict__))
self.event_verified.set()
def start_event_transform_listener(self):
es = EventSubscriber(event_type=OT.DeviceStatusAlertEvent,
callback=self.validate_transform_event)
es.start()
self.addCleanup(es.stop)
def test_download(self):
egg_url = 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
egg_path = TransformWorker.download_egg(egg_url)
import pkg_resources
pkg_resources.working_set.add_entry(egg_path)
from ion_example.add_arrays import add_arrays
a = add_arrays(1, 2)
self.assertEquals(a, 3)
def test_lookup_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsubcli.create_stream_definition(
'lookup', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id)
data_product = DataProduct(name='lookup data product')
tdom, sdom = time_series_domain()
data_product.temporal_domain = tdom.dump()
data_product.spatial_domain = sdom.dump()
data_product_id = self.dpsc_cli.create_data_product(
data_product, stream_definition_id=stream_def_id)
self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id)
data_producer = DataProducer(name='producer')
data_producer.producer_context = DataProcessProducerContext()
data_producer.producer_context.configuration['qc_keys'] = [
'offset_document'
]
data_producer_id, _ = self.rrclient.create(data_producer)
self.addCleanup(self.rrclient.delete, data_producer_id)
assoc, _ = self.rrclient.create_association(
subject=data_product_id,
object=data_producer_id,
predicate=PRED.hasDataProducer)
self.addCleanup(self.rrclient.delete_association, assoc)
document_keys = self.damsclient.list_qc_references(data_product_id)
self.assertEquals(document_keys, ['offset_document'])
svm = StoredValueManager(self.container)
svm.stored_value_cas('offset_document', {'offset_a': 2.0})
self.dpsc_cli.activate_data_product_persistence(data_product_id)
dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id,
predicate=PRED.hasDataset,
id_only=True)
dataset_id = dataset_ids[0]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20.]
granule = rdt.to_granule()
stream_ids, _ = self.rrclient.find_objects(subject=data_product_id,
predicate=PRED.hasStream,
id_only=True)
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp'], rdt2['temp'])
np.testing.assert_array_almost_equal(rdt2['calibrated'],
np.array([22.0]))
svm.stored_value_cas('updated_document', {'offset_a': 3.0})
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent)
ep.publish_event(origin=data_product_id,
reference_keys=['updated_document'])
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1]
rdt['temp'] = [20.]
granule = rdt.to_granule()
gevent.sleep(2) # Yield so that the event goes through
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt2['temp'], np.array([20., 20.]))
np.testing.assert_array_almost_equal(rdt2['calibrated'],
np.array([22.0, 23.0]))
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
self.pdict_id = self.ph.create_simple_qc_pdict()
self.svm = StoredValueManager(self.container)
def test_pydap(self):
if not CFG.get_safe('bootstrap.use_pydap', False):
raise unittest.SkipTest('PyDAP is off (bootstrap.use_pydap)')
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 10)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
pydap_host = CFG.get_safe('server.pydap.host', 'localhost')
pydap_port = CFG.get_safe('server.pydap.port', 8001)
url = 'http://%s:%s/%s' % (pydap_host, pydap_port, dataset_id)
for i in xrange(
3
): # Do it three times to test that the cache doesn't corrupt the requests/responses
ds = open_url(url)
np.testing.assert_array_equal(ds['time'][:], np.arange(10))
untested = []
for k, v in rdt.iteritems():
if k == rdt.temporal_parameter:
continue
context = rdt.context(k)
if isinstance(context.param_type, QuantityType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, ArrayType):
if context.param_type.inner_encoding is None:
values = np.empty(rdt[k].shape, dtype='O')
for i, obj in enumerate(rdt[k]):
values[i] = str(obj)
np.testing.assert_array_equal(ds[k][k][:][0], values)
elif len(rdt[k].shape) > 1:
values = np.empty(rdt[k].shape[0], dtype='O')
for i in xrange(rdt[k].shape[0]):
values[i] = ','.join(
map(lambda x: str(x), rdt[k][i].tolist()))
elif isinstance(context.param_type, ConstantType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, CategoryType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
else:
untested.append('%s (%s)' % (k, context.param_type))
if untested:
raise AssertionError('Untested parameters: %s' % untested)
def test_lookup_values_ingest_replay(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsub_management.create_stream_definition(
'lookups', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
stream_id, route = self.pubsub_management.create_stream(
'example',
exchange_point=self.exchange_point_name,
stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
config = DotDict()
config.process.lookup_docs = ['test1', 'test2']
self.ingestion_management.persist_data_stream(
stream_id=stream_id,
ingestion_configuration_id=ingestion_config_id,
dataset_id=dataset_id,
config=config)
self.addCleanup(self.ingestion_management.unpersist_data_stream,
stream_id, ingestion_config_id)
stored_value_manager = StoredValueManager(self.container)
stored_value_manager.stored_value_cas('test1', {
'offset_a': 10.0,
'offset_b': 13.1
})
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = [20.0] * 20
granule = rdt.to_granule()
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(20))
np.testing.assert_array_almost_equal(rdt_out['temp'],
np.array([20.] * 20))
np.testing.assert_array_almost_equal(rdt_out['calibrated'],
np.array([30.] * 20))
np.testing.assert_array_equal(
rdt_out['offset_b'],
np.array([rdt_out.fill_value('offset_b')] * 20))
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(20, 40)
rdt['temp'] = [20.0] * 20
granule = rdt.to_granule()
dataset_monitor.event.clear()
stored_value_manager.stored_value_cas('test1', {'offset_a': 20.0})
stored_value_manager.stored_value_cas('coefficient_document',
{'offset_b': 10.0})
gevent.sleep(2)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(40))
np.testing.assert_array_almost_equal(rdt_out['temp'],
np.array([20.] * 20 + [20.] * 20))
np.testing.assert_array_equal(rdt_out['offset_b'],
np.array([10.] * 40))
np.testing.assert_array_almost_equal(rdt_out['calibrated'],
np.array([30.] * 20 + [40.] * 20))
np.testing.assert_array_almost_equal(rdt_out['calibrated_b'],
np.array([40.] * 20 + [50.] * 20))
def check_vel3d_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(
stream_def_id)
self.assertEquals(pdict.name, 'vel3d_b_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
lat = 14.6846
lon = -51.044
ts = np.array([
3319563600, 3319567200, 3319570800, 3319574400, 3319578000,
3319581600, 3319585200, 3319588800, 3319592400, 3319596000
],
dtype=np.float)
ve = np.array([-3.2, 0.1, 0., 2.3, -0.1, 5.6, 5.1, 5.8, 8.8, 10.3])
vn = np.array([18.2, 9.9, 12., 6.6, 7.4, 3.4, -2.6, 0.2, -1.5, 4.1])
vu = np.array([-1.1, -0.6, -1.4, -2, -1.7, -2, 1.3, -1.6, -1.1, -4.5])
ve_expected = np.array([
-0.085136, -0.028752, -0.036007, 0.002136, -0.023158, 0.043218,
0.056451, 0.054727, 0.088446, 0.085952
])
vn_expected = np.array([
0.164012, 0.094738, 0.114471, 0.06986, 0.07029, 0.049237,
-0.009499, 0.019311, 0.012096, 0.070017
])
vu_expected = np.array([
-0.011, -0.006, -0.014, -0.02, -0.017, -0.02, 0.013, -0.016,
-0.011, -0.045
])
rdt['time'] = ts
rdt['lat'] = [lat] * 10
rdt['lon'] = [lon] * 10
rdt['turbulent_velocity_east'] = ve
rdt['turbulent_velocity_north'] = vn
rdt['turbulent_velocity_up'] = vu
passing &= self.assert_array_almost_equal(
rdt['eastward_turbulent_velocity'], ve_expected)
passing &= self.assert_array_almost_equal(
rdt['northward_turbulent_velocity'], vn_expected)
passing &= self.assert_array_almost_equal(
rdt['upward_turbulent_velocity'], vu_expected)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(
rdt['eastward_turbulent_velocity'], ve_expected)
passing &= self.assert_array_almost_equal(
rdt['northward_turbulent_velocity'], vn_expected)
passing &= self.assert_array_almost_equal(
rdt['upward_turbulent_velocity'], vu_expected)
return passing
def check_trhph_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(
stream_def_id)
passing &= self.assertEquals(pdict.name, 'trhph_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
# calibration constants
a = 1.98e-9
b = -2.45e-6
c = 9.28e-4
d = -0.0888
e = 0.731
V_s = 1.506
V_c = 0.
T = 11.8
r1 = 0.906
r2 = 4.095
r3 = 4.095
ORP_V = 1.806
Cl = np.nan
offset = 2008
gain = 4.0
# Normally this would be 50 per the DPS but the precision is %4.0f which truncates the values to the nearest 1...
ORP = ((ORP_V * 1000.) - offset) / gain
ntp_now = time.time() + 2208988800
rdt['cc_a'] = [a]
rdt['cc_b'] = [b]
rdt['cc_c'] = [c]
rdt['cc_d'] = [d]
rdt['cc_e'] = [e]
rdt['ref_temp_volts'] = [V_s]
rdt['resistivity_temp_volts'] = [V_c]
rdt['eh_sensor'] = [ORP_V]
rdt['resistivity_5'] = [r1]
rdt['resistivity_x1'] = [r2]
rdt['resistivity_x5'] = [r3]
rdt['cc_offset'] = [offset]
rdt['cc_gain'] = [gain]
rdt['time'] = [ntp_now]
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'],
[T], 2)
passing &= self.assert_array_almost_equal(
rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP],
4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'],
[T], 2)
passing &= self.assert_array_almost_equal(
rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP],
4)
return passing
class TestQCFunctions(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
self.pdict_id = self.ph.create_simple_qc_pdict()
self.svm = StoredValueManager(self.container)
def new_rdt(self,ref='QCTEST'):
self.stream_def_id = self.create_stream_definition(uuid4().hex, parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
def test_qc_functions(self):
self.check_global_range()
self.check_spike()
self.check_stuck_value()
self.check_trend()
self.check_gradient()
self.check_localrange()
self.check_propagate()
def check_global_range(self):
log.info('check_global_range')
self.new_rdt()
self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_spike(self):
log.info('check_spike')
self.new_rdt()
self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1])
def check_stuck_value(self):
log.info('check_stuck_value')
self.new_rdt()
self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
def check_trend(self):
log.info('check_trend')
self.new_rdt()
self.svm.stored_value_cas('trend_QCTEST_TEMPWAT', {'time_interval':0, 'polynomial_order': 1, 'standard_deviation': 3})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [0.8147, 0.9058, 0.1270, 0.9134, 0.6324, 0.0975, 0.2785, 0.5469, 0.9575, 0.9649]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_trndtst_qc'], [1] * 10)
def check_propagate(self):
log.info('check_propagate')
self.new_rdt()
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt['tempwat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
np.testing.assert_array_equal(self.rdt['cmbnflg_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_gradient(self):
log.info('check_gradient')
self.new_rdt()
self.svm.stored_value_cas('grad_QCTEST_TEMPWAT_time', {'d_dat_dx': 50, 'min_dx': 0, 'start_dat': 0, 'tol_dat': 5})
self.rdt['time'] = np.arange(5)
self.rdt['temp'] = [3, 5, 98, 99, 4]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_gradtst_qc'], [1, 1, 0, 0, 1])
def check_localrange(self):
log.info('check_localrange')
self.new_rdt()
t = np.array([3580144703.7555027, 3580144704.7555027, 3580144705.7555027, 3580144706.7555027, 3580144707.7555027, 3580144708.7555027, 3580144709.7555027, 3580144710.7555027, 3580144711.7555027, 3580144712.7555027])
pressure = np.random.rand(10) * 2 + 33.0
t_v = ntp_to_month(t)
dat = t_v + pressure + np.arange(16,26)
def lim1(p,m):
return p+m+10
def lim2(p,m):
return p+m+20
pressure_grid, month_grid = np.meshgrid(np.arange(0,150,10), np.arange(11))
points = np.column_stack([pressure_grid.flatten(), month_grid.flatten()])
datlim_0 = lim1(points[:,0], points[:,1])
datlim_1 = lim2(points[:,0], points[:,1])
datlim = np.column_stack([datlim_0, datlim_1])
datlimz = points
self.svm.stored_value_cas('lrt_QCTEST_TEMPWAT', {'datlim':datlim.tolist(), 'datlimz':datlimz.tolist(), 'dims':['pressure', 'month']})
self.rdt['time'] = t
self.rdt['temp'] = dat
self.rdt['pressure'] = pressure
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1 ,1 ,1 ,1 ,1 ,0 ,0 ,0 ,0 ,0])
def test_qc_interval_integration(self):
# 1 need to make a dataset that only has one discrete qc violation
# 2 Launch the process
# 3 Setup the scheduler to run it say three times
# 4 Get the Events and verify the data
#--------------------------------------------------------------------------------
# Make a dataset that has only one discrete qc violation
#--------------------------------------------------------------------------------
dp_id, dataset_id, stream_def_id = self.make_data_product()
ph = ParameterHelper(self.dataset_management, self.addCleanup)
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
for rdt in self.populate_vectors(stream_def_id, 1,
lambda x: [41] + [39] * (x - 1)):
ph.publish_rdt_to_data_product(dp_id, rdt)
self.assertTrue(
monitor.event.wait(10)) # Give it 10 seconds to populate
#--------------------------------------------------------------------------------
# Launch the process
#--------------------------------------------------------------------------------
interval_key = uuid4().hex
config = DotDict()
config.process.interval_key = interval_key
config.process.qc_params = [
'glblrng_qc'
] # The others are tested in other tests for completeness
self.sync_launch(config)
async_queue = Queue()
def callback(event, *args, **kwargs):
times = event.temporal_values
self.assertEquals(len(times), 1)
async_queue.put(1)
es = EventSubscriber(event_type=OT.ParameterQCEvent,
origin=dp_id,
callback=callback,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
#--------------------------------------------------------------------------------
# Setup the scheduler
#--------------------------------------------------------------------------------
timer_id = self.scheduler_service.create_interval_timer(
start_time=time.time(),
end_time=time.time() + 13,
interval=5,
event_origin=interval_key)
#--------------------------------------------------------------------------------
# Get the events and verify them
#--------------------------------------------------------------------------------
try:
for i in xrange(2):
async_queue.get(timeout=10)
except Empty:
raise AssertionError('QC Events not raised')
def test_sparse_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_sparse()
stream_def_id = self.pubsub_management.create_stream_definition(
'sparse', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
stream_id, route = self.pubsub_management.create_stream(
'example',
exchange_point=self.exchange_point_name,
stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
dataset_id = self.create_dataset(pdict_id)
self.start_ingestion(stream_id, dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
ntp_now = time.time() + 2208988800
rdt = ph.get_rdt(stream_def_id)
rdt['time'] = [ntp_now]
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [300000]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = [None]
rdt['lat'] = [45]
rdt['conductivity'] = [4341400]
rdt['driver_timestamp'] = [ntp_now]
rdt['lon'] = [-71]
rdt['pressure'] = [256.8]
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.event.wait(30))
dataset_monitor.event.clear()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], rdt['time'])
np.testing.assert_array_almost_equal(rdt_out['temp'], rdt['temp'])
np.testing.assert_array_almost_equal(rdt_out['lat'], np.array([45]))
np.testing.assert_array_almost_equal(rdt_out['lon'], np.array([-71]))
np.testing.assert_array_almost_equal(rdt_out['conductivity_L1'],
np.array([42.914]))
np.testing.assert_array_almost_equal(rdt_out['temp_L1'],
np.array([20.]))
np.testing.assert_array_almost_equal(rdt_out['pressure_L1'],
np.array([3.068]))
np.testing.assert_array_almost_equal(
rdt_out['density'], np.array([1021.7144739593881],
dtype='float32'))
np.testing.assert_array_almost_equal(
rdt_out['salinity'], np.array([30.935132729668283],
dtype='float32'))
rdt = ph.get_rdt(stream_def_id)
rdt['lat'] = [46]
rdt['lon'] = [-73]
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.event.wait(30))
dataset_monitor.event.clear()
rdt = ph.get_rdt(stream_def_id)
rdt['lat'] = [1000]
rdt['lon'] = [3]
publisher.publish(rdt.to_granule())
rdt = ph.get_rdt(stream_def_id)
rdt['time'] = [ntp_now]
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [300000]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = [None]
rdt['conductivity'] = [4341400]
rdt['driver_timestamp'] = [ntp_now]
rdt['pressure'] = [256.8]
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.event.wait(30))
dataset_monitor.event.clear()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['lat'], np.array([45,
46]))
np.testing.assert_array_almost_equal(rdt_out['lon'],
np.array([-71, -73]))
def test_sparse_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_sparse()
stream_def_id = self.pubsub_management.create_stream_definition('sparse', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
dataset_id = self.create_dataset(pdict_id)
self.start_ingestion(stream_id,dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
# Publish initial granule
# the first one has the sparse value set inside it, sets lat to 45 and lon to -71
ntp_now = time.time() + 2208988800
rdt = ph.get_rdt(stream_def_id)
rdt['time'] = [ntp_now]
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [300000]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = ['']
rdt['lat'] = [45]
rdt['conductivity'] = [4341400]
rdt['driver_timestamp'] = [ntp_now]
rdt['lon'] = [-71]
rdt['pressure'] = [256.8]
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
dataset_monitor.reset()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
# Check the values and make sure they're correct
np.testing.assert_allclose(rdt_out['time'], rdt['time'])
np.testing.assert_allclose(rdt_out['temp'], rdt['temp'])
np.testing.assert_allclose(rdt_out['lat'], np.array([45]))
np.testing.assert_allclose(rdt_out['lon'], np.array([-71]))
np.testing.assert_allclose(rdt_out['conductivity_L1'], np.array([42.914]))
np.testing.assert_allclose(rdt_out['temp_L1'], np.array([20.]))
np.testing.assert_allclose(rdt_out['pressure_L1'], np.array([3.068]))
np.testing.assert_allclose(rdt_out['density'], np.array([1021.7144739593881], dtype='float32'))
np.testing.assert_allclose(rdt_out['salinity'], np.array([30.935132729668283], dtype='float32'))
# We're going to change the lat/lon
rdt = ph.get_rdt(stream_def_id)
rdt['time'] = time.time() + 2208988800
rdt['lat'] = [46]
rdt['lon'] = [-73]
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
dataset_monitor.reset()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_allclose(rdt_out['time'], rdt['time'])
for i in xrange(9):
ntp_now = time.time() + 2208988800
rdt['time'] = [ntp_now]
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [300000]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = [None]
rdt['conductivity'] = [4341400]
rdt['driver_timestamp'] = [ntp_now]
rdt['pressure'] = [256.8]
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
dataset_monitor.reset()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_allclose(rdt_out['pressure'], np.array([256.8] * 10))
np.testing.assert_allclose(rdt_out['lat'], np.array([45] + [46] * 9))
np.testing.assert_allclose(rdt_out['lon'], np.array([-71] + [-73] * 9))
class TestDMExtended(DMTestCase):
'''
ion/services/dm/test/test_dm_extended.py:TestDMExtended
'''
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
@attr('UTIL')
def test_pydap_handlers(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
stream_def_id = self.create_stream_definition('ctd', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('ctd', stream_def_id=stream_def_id)
self.activate_data_product(data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
rdt = self.ph.get_rdt(stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id,rdt)
dataset_monitor.event.wait(10)
from pydap.client import open_url
pydap_host = CFG.get_safe('server.pydap.host','localhost')
pydap_port = CFG.get_safe('server.pydap.port',8001)
url = 'http://%s:%s/%s' %(pydap_host, pydap_port, dataset_id)
ds = open_url(url)
ds['temp']['temp'][:]
def make_array_data_product(self):
pdict_id = self.ph.crete_simple_array_pdict()
stream_def_id = self.create_stream_definition('test_array_flow_paths', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('test_array_flow_paths', stream_def_id)
self.activate_data_product(data_product_id)
return data_product_id, stream_def_id
def preload_beta(self):
config = DotDict()
config.op = 'load'
config.loadui=True
config.ui_path = "https://userexperience.oceanobservatories.org/database-exports/Candidates"
config.attachments = "res/preload/r2_ioc/attachments"
config.scenario = 'BETA'
config.categories='ParameterFunctions,ParameterDefs,ParameterDictionary'
self.container.spawn_process('preloader', 'ion.processes.bootstrap.ion_loader', 'IONLoader', config)
def create_google_dt_workflow_def(self):
# Check to see if the workflow defnition already exist
workflow_def_ids,_ = self.resource_registry.find_resources(restype=RT.WorkflowDefinition, name='Realtime_Google_DT', id_only=True)
if len(workflow_def_ids) > 0:
workflow_def_id = workflow_def_ids[0]
else:
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Realtime_Google_DT',description='Convert stream data to Google Datatable')
#Add a transformation process definition
google_dt_procdef_id = self.create_google_dt_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflow_management.create_workflow_definition(workflow_def_obj)
return workflow_def_id
def create_google_dt_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.resource_registry.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform')
if len(dpd) > 0:
return dpd[0]
# Data Process Definition
dpd_obj = IonObject(RT.DataProcessDefinition,
name='google_dt_transform',
description='Convert data streams to Google DataTables',
module='ion.processes.data.transforms.viz.google_dt',
class_name='VizTransformGoogleDT')
try:
procdef_id = self.data_process_management.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new VizTransformGoogleDT data process definition: %s" %ex)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('google_dt', id_only=True)
# create a stream definition for the data from the
stream_def_id = self.pubsub_management.create_stream_definition(name='VizTransformGoogleDT', parameter_dictionary_id=pdict_id)
self.data_process_management.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id, binding='google_dt' )
return procdef_id
@attr('UTIL')
def test_dm_realtime_visualization(self):
self.preload_beta()
# Create the google_dt workflow definition since there is no preload for the test
workflow_def_id = self.create_google_dt_workflow_def()
#Create the input data product
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_simulator', id_only=True)
stream_def_id = self.create_stream_definition('ctd sim L2', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('ctd simulator', stream_def_id=stream_def_id)
self.activate_data_product(data_product_id)
#viz_token = self.visualization.initiate_realtime_visualization_data(data_product_id=data_product_id)
streamer = Streamer(data_product_id)
self.addCleanup(streamer.stop)
breakpoint(locals())
# ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product()
# ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id)
#
# vis_params ={}
# vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id, visualization_parameters=simplejson.dumps(vis_params))
# print ">>>>>>>>>>>>>>>>>>> vis_token_resp : ", vis_token_resp
#
# import ast
# vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"]
#
# result = gevent.event.AsyncResult()
#
# def get_vis_messages(get_data_count=7): #SHould be an odd number for round robbin processing by service workers
#
#
# get_cnt = 0
# while get_cnt < get_data_count:
#
# vis_data = self.vis_client.get_realtime_visualization_data(vis_token)
# if (vis_data):
# self.validate_google_dt_transform_results(vis_data)
#
# get_cnt += 1
# gevent.sleep(5) # simulates the polling from UI
#
# result.set(get_cnt)
#
# gevent.spawn(get_vis_messages)
#
# result.get(timeout=90)
#
# #Trying to continue to receive messages in the queue
# gevent.sleep(2.0) # Send some messages - don't care how many
#
#
# # Cleanup
# self.vis_client.terminate_realtime_visualization_data(vis_token)
#
#
# #Turning off after everything - since it is more representative of an always on stream of data!
# self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
@attr('INT',group='dm')
def test_array_visualization(self):
data_product_id, stream_def_id = self.make_array_data_product()
# Make a granule with an array type, give it a few values
# Send it to google_dt transform, verify output
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(2208988800, 2208988810)
rdt['temp_sample'] = np.arange(10*4).reshape(10,4)
rdt['cond_sample'] = np.arange(10*4).reshape(10,4)
granule = rdt.to_granule()
dataset_monitor = DatasetMonitor(self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id))
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
dataset_monitor.event.wait(10)
gdt_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True)
gdt_stream_def = self.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id)
gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(granule, params=gdt_stream_def)
rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule)
testval = {'data_content': [
[0.0 , 0.0 , 1.0 , 2.0 , 3.0 , 0.0 , 2.0 , 4.0 , 6.0 , 0.0 , 1.0 , 2.0 , 3.0] ,
[1.0 , 4.0 , 5.0 , 6.0 , 7.0 , 8.0 , 10.0 , 12.0 , 14.0 , 4.0 , 5.0 , 6.0 , 7.0] ,
[2.0 , 8.0 , 9.0 , 10.0 , 11.0 , 16.0 , 18.0 , 20.0 , 22.0 , 8.0 , 9.0 , 10.0 , 11.0] ,
[3.0 , 12.0 , 13.0 , 14.0 , 15.0 , 24.0 , 26.0 , 28.0 , 30.0 , 12.0 , 13.0 , 14.0 , 15.0] ,
[4.0 , 16.0 , 17.0 , 18.0 , 19.0 , 32.0 , 34.0 , 36.0 , 38.0 , 16.0 , 17.0 , 18.0 , 19.0] ,
[5.0 , 20.0 , 21.0 , 22.0 , 23.0 , 40.0 , 42.0 , 44.0 , 46.0 , 20.0 , 21.0 , 22.0 , 23.0] ,
[6.0 , 24.0 , 25.0 , 26.0 , 27.0 , 48.0 , 50.0 , 52.0 , 54.0 , 24.0 , 25.0 , 26.0 , 27.0] ,
[7.0 , 28.0 , 29.0 , 30.0 , 31.0 , 56.0 , 58.0 , 60.0 , 62.0 , 28.0 , 29.0 , 30.0 , 31.0] ,
[8.0 , 32.0 , 33.0 , 34.0 , 35.0 , 64.0 , 66.0 , 68.0 , 70.0 , 32.0 , 33.0 , 34.0 , 35.0] ,
[9.0 , 36.0 , 37.0 , 38.0 , 39.0 , 72.0 , 74.0 , 76.0 , 78.0 , 36.0 , 37.0 , 38.0 , 39.0]] ,
'data_description': [('time', 'number', 'time'),
('temp_sample[0]', 'number', 'temp_sample[0]', {'precision': '5'}),
('temp_sample[1]', 'number', 'temp_sample[1]', {'precision': '5'}),
('temp_sample[2]', 'number', 'temp_sample[2]', {'precision': '5'}),
('temp_sample[3]', 'number', 'temp_sample[3]', {'precision': '5'}),
('temp_offset[0]', 'number', 'temp_offset[0]', {'precision': '5'}),
('temp_offset[1]', 'number', 'temp_offset[1]', {'precision': '5'}),
('temp_offset[2]', 'number', 'temp_offset[2]', {'precision': '5'}),
('temp_offset[3]', 'number', 'temp_offset[3]', {'precision': '5'}),
('cond_sample[0]', 'number', 'cond_sample[0]', {'precision': '5'}),
('cond_sample[1]', 'number', 'cond_sample[1]', {'precision': '5'}),
('cond_sample[2]', 'number', 'cond_sample[2]', {'precision': '5'}),
('cond_sample[3]', 'number', 'cond_sample[3]', {'precision': '5'})],
'viz_product_type': 'google_dt'}
self.assertEquals(rdt['google_dt_components'][0], testval)
@attr('INT',group='dm')
def test_array_flow_paths(self):
data_product_id, stream_def_id = self.make_array_data_product()
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
dm = DatasetMonitor(dataset_id)
self.addCleanup(dm.stop)
# I need to make sure that we can fill the RDT with its values
# Test for one timestep
# Test for multiple timesteps
# Publishes
# Ingests correctly
# Retrieves correctly
#--------------------------------------------------------------------------------
# Ensure that the RDT can be filled with ArrayType values
#--------------------------------------------------------------------------------
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp_sample'] = [[0,1,2,3,4]]
np.testing.assert_array_equal(rdt['temp_sample'], np.array([[0,1,2,3,4]]))
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dm.event.wait(10))
dm.event.clear()
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp_sample'], np.array([[0,1,2,3,4]]))
#--------------------------------------------------------------------------------
# Ensure that it deals with multiple values
#--------------------------------------------------------------------------------
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1,2,3]
rdt['temp_sample'] = [[0,1,2,3,4],[1],[5,5,5,5,5]]
m = rdt.fill_value('temp_sample') or np.finfo(np.float32).max
np.testing.assert_equal(m,np.finfo(np.float32).max)
np.testing.assert_array_equal(rdt['temp_sample'], [[0,1,2,3,4],[1,m,m,m,m],[5,5,5,5,5]])
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dm.event.wait(10))
dm.event.clear()
#--------------------------------------------------------------------------------
# Retrieve and Verify
#--------------------------------------------------------------------------------
retrieved_granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(retrieved_granule)
np.testing.assert_array_equal(rdt['time'], np.array([0,1,2,3]))
np.testing.assert_array_equal(rdt['temp_sample'], np.array([[0,1,2,3,4],[0,1,2,3,4],[1,m,m,m,m],[5,5,5,5,5]]))
@attr('UTIL')
def test_creation_args(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
stream_def_id = self.create_stream_definition('ctd', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('ctd', stream_def_id=stream_def_id)
self.activate_data_product(data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
breakpoint(locals())
rdt = self.ph.get_rdt(stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id,rdt)
dataset_monitor.event.wait(10)
breakpoint(locals())
granule = self.data_retriever.retrieve(dataset_id)
breakpoint(locals())