class StoredValueTransform(TransformStreamListener):
'''
Receives granules from a stream and persists the latest value in the object store
Background:
Platforms publish geospatial information on a separate stream from the
instruments. Various data products require geospatial information about
the instrument to calculate the variables. This component persists the
latest value in a simple data storage container where complex data
containers can access it.
'''
def on_start(self):
TransformStreamListener.on_start(self)
self.document_key = self.CFG.get_safe('process.document_key')
self.stored_value_manager = StoredValueManager(self.container)
def recv_packet(self, msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
document = {}
for k,v in rdt.iteritems():
value_array = np.atleast_1d(v[:])
if 'f' in value_array.dtype.str:
document[k] = float(value_array[-1])
elif 'i' in value_array.dtype.str:
document[k] = int(value_array[-1])
self.stored_value_manager.stored_value_cas(self.document_key, document)
class StoredValueTransform(TransformStreamListener):
'''
Receives granules from a stream and persists the latest value in the object store
Background:
Platforms publish geospatial information on a separate stream from the
instruments. Various data products require geospatial information about
the instrument to calculate the variables. This component persists the
latest value in a simple data storage container where complex data
containers can access it.
'''
def on_start(self):
TransformStreamListener.on_start(self)
self.document_key = self.CFG.get_safe('process.document_key')
self.stored_value_manager = StoredValueManager(self.container)
def recv_packet(self, msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
document = {}
for k, v in rdt.iteritems():
value_array = np.atleast_1d(v[:])
if 'f' in value_array.dtype.str:
document[k] = float(value_array[-1])
elif 'i' in value_array.dtype.str:
document[k] = int(value_array[-1])
self.stored_value_manager.stored_value_cas(self.document_key, document)
def parse_document(cls, container, parser, document_path):
svm = StoredValueManager(container)
document = ''
with open(document_path,'r') as f:
document = f.read()
for k,v in parser(document):
svm.stored_value_cas(k,v)
return
def parse_document(cls, container, parser, document_path):
svm = StoredValueManager(container)
document = ''
with open(document_path, 'r') as f:
document = f.read()
for k, v in parser(document):
svm.stored_value_cas(k, v)
return
def test_global_range_test_parser(self):
svm = StoredValueManager(self.container)
for key,doc in grt_parser(grt_sample_doc):
svm.stored_value_cas(key,doc)
self.addCleanup(svm.delete_stored_value,key)
doc = svm.read_value('grt_sbe37-abc123_TEMPWAT_UHHH')
self.assertEquals(doc['grt_min_value'], 0.)
self.assertEquals(doc['array'],'array 1')
doc = svm.read_value('grt_sbe37-abc123_PRESWAT_flagged')
self.assertEquals(doc['grt_max_value'], 689.47)
def test_gradient_test(self):
svm = StoredValueManager(self.container)
for key,doc in gradient_test_parser(gradient_test_sample_doc):
svm.stored_value_cas(key,doc)
self.addCleanup(svm.delete_stored_value,key)
doc = svm.read_value('grad_CHEL-C-32-12_PRESSURE_DENSITY')
self.assertEquals(doc['units_dat'], 'dbar')
self.assertEquals(doc['d_dat_dx'], 2.781)
doc = svm.read_value('grad_CHEW-BA-C-A12_PRESSURE_DENSITY')
self.assertEquals(doc['units_x'], 'kg m-3')
self.assertEquals(doc['tol_dat'], 12.)
def test_stuck_value_test(self):
svm = StoredValueManager(self.container)
for key, doc in trend_parser(trend_value_test_sample_doc):
svm.stored_value_cas(key, doc)
self.addCleanup(svm.delete_stored_value, key)
doc = svm.read_value("trend_ssxbt-ssn719_PRESSURE")
self.assertEquals(doc["time_interval"], 1.0)
self.assertEquals(doc["standard_deviation"], 0.0)
doc = svm.read_value("trend_ssxbt-ssn719_COND")
self.assertEquals(doc["time_interval"], 3.0)
self.assertEquals(doc["polynomial_order"], "third")
def test_stuck_value_test(self):
svm = StoredValueManager(self.container)
for key,doc in stuck_value_test_parser(stuck_value_test_sample_doc):
svm.stored_value_cas(key,doc)
self.addCleanup(svm.delete_stored_value,key)
doc = svm.read_value('svt_ssxbt-ssn719_PRESSURE')
self.assertEquals(doc['svt_resolution'], 1.2)
self.assertEquals(doc['svt_n'], 10.)
doc = svm.read_value('svt_ssxbt-ssn719_COND')
self.assertEquals(doc['svt_resolution'], 0.2)
self.assertEquals(doc['units'], 'S m-1')
def test_basic_stored_docs(self):
doc = {'key':'value'}
doc_key = 'doc_id'
svm = StoredValueManager(self.container)
doc_id, rev = svm.stored_value_cas(doc_key, doc)
self.addCleanup(svm.delete_stored_value, doc_key)
doc = svm.read_value(doc_key)
self.assertTrue('key' in doc and doc['key']=='value')
svm.stored_value_cas(doc_key,{'key2':'value2'})
doc = svm.read_value(doc_key)
self.assertTrue('key' in doc and doc['key']=='value')
self.assertTrue('key2' in doc and doc['key2']=='value2')
def test_rdt_lookup(self):
rdt = self.create_lookup_rdt()
self.assertTrue('offset_a' in rdt.lookup_values())
self.assertFalse('offset_b' in rdt.lookup_values())
rdt['time'] = [0]
rdt['temp'] = [10.0]
rdt['offset_a'] = [2.0]
self.assertEquals(rdt['offset_b'], None)
self.assertEquals(rdt.lookup_values(), ['offset_a'])
np.testing.assert_array_almost_equal(rdt['calibrated'], np.array([12.0]))
svm = StoredValueManager(self.container)
svm.stored_value_cas('coefficient_document', {'offset_b':2.0})
rdt.fetch_lookup_values()
np.testing.assert_array_equal(rdt['offset_b'], np.array([2.0]))
np.testing.assert_array_equal(rdt['calibrated_b'], np.array([14.0]))
def test_rdt_lookup(self):
rdt = self.create_lookup_rdt()
self.assertTrue('offset_a' in rdt.lookup_values())
self.assertFalse('offset_b' in rdt.lookup_values())
rdt['time'] = [0]
rdt['temp'] = [10.0]
rdt['offset_a'] = [2.0]
self.assertEquals(rdt['offset_b'], None)
self.assertEquals(rdt.lookup_values(), ['offset_a'])
np.testing.assert_array_almost_equal(rdt['calibrated'], np.array([12.0]))
svm = StoredValueManager(self.container)
svm.stored_value_cas('coefficient_document', {'offset_b':2.0})
svm.stored_value_cas("GA03FLMA-RI001-13-CTDMOG999_OFFSETC", {'offset_c':3.0})
rdt.fetch_lookup_values()
np.testing.assert_array_equal(rdt['offset_b'], np.array([2.0]))
np.testing.assert_array_equal(rdt['calibrated_b'], np.array([14.0]))
np.testing.assert_array_equal(rdt['offset_c'], np.array([3.0]))
def test_global_range_lookup(self):
reference_designator = "CE01ISSM-MF005-01-CTDBPC999"
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_simple_qc_pdict()
svm = StoredValueManager(self.container)
doc_key = 'grt_%s_TEMPWAT' % reference_designator
svm.stored_value_cas(doc_key, {'grt_min_value':-2, 'grt_max_value':40})
stream_def_id = self.pubsub_management.create_stream_definition('qc parsed', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':reference_designator})
self.addCleanup(self.pubsub_management.delete_stream_definition,stream_def_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20]
rdt.fetch_lookup_values()
min_field = [i for i in rdt.fields if 'grt_min_value' in i][0]
max_field = [i for i in rdt.fields if 'grt_max_value' in i][0]
np.testing.assert_array_almost_equal(rdt[min_field], [-2.])
np.testing.assert_array_almost_equal(rdt[max_field], [40.])
np.testing.assert_array_almost_equal(rdt['tempwat_glblrng_qc'],[1])
def populate_qc_tables(self):
svm = StoredValueManager(self.container)
svm.stored_value_cas('grt_QCTEST_TEMPWAT', {
'grt_min_value': -2.,
'grt_max_value': 40.
})
svm.stored_value_cas('svt_QCTEST_TEMPWAT', {
'svt_resolution': 0.001,
'svt_n': 4
})
svm.stored_value_cas('spike_QCTEST_TEMPWAT', {
'acc': 0.1,
'spike_n': 5,
'spike_l': 5
})
class TestQCFunctions(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = self.ph.create_simple_qc_pdict()
self.stream_def_id = self.pubsub_management.create_stream_definition('global range', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.addCleanup(self.pubsub_management.delete_stream_definition, self.stream_def_id)
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.svm = StoredValueManager(self.container)
def test_global_range_test(self):
self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def test_spike_test(self): # I know how redundant this sounds
self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1])
def test_stuck_value_test(self):
self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
def test_lookup_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsubcli.create_stream_definition(
'lookup', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id)
data_product = DataProduct(name='lookup data product')
tdom, sdom = time_series_domain()
data_product.temporal_domain = tdom.dump()
data_product.spatial_domain = sdom.dump()
data_product_id = self.dpsc_cli.create_data_product(
data_product, stream_definition_id=stream_def_id)
self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id)
data_producer = DataProducer(name='producer')
data_producer.producer_context = DataProcessProducerContext()
data_producer.producer_context.configuration['qc_keys'] = [
'offset_document'
]
data_producer_id, _ = self.rrclient.create(data_producer)
self.addCleanup(self.rrclient.delete, data_producer_id)
assoc, _ = self.rrclient.create_association(
subject=data_product_id,
object=data_producer_id,
predicate=PRED.hasDataProducer)
self.addCleanup(self.rrclient.delete_association, assoc)
document_keys = self.damsclient.list_qc_references(data_product_id)
self.assertEquals(document_keys, ['offset_document'])
svm = StoredValueManager(self.container)
svm.stored_value_cas('offset_document', {'offset_a': 2.0})
self.dpsc_cli.activate_data_product_persistence(data_product_id)
dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id,
predicate=PRED.hasDataset,
id_only=True)
dataset_id = dataset_ids[0]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20.]
granule = rdt.to_granule()
stream_ids, _ = self.rrclient.find_objects(subject=data_product_id,
predicate=PRED.hasStream,
id_only=True)
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp'], rdt2['temp'])
np.testing.assert_array_almost_equal(rdt2['calibrated'],
np.array([22.0]))
svm.stored_value_cas('updated_document', {'offset_a': 3.0})
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent)
ep.publish_event(origin=data_product_id,
reference_keys=['updated_document'])
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1]
rdt['temp'] = [20.]
granule = rdt.to_granule()
gevent.sleep(2) # Yield so that the event goes through
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt2['temp'], np.array([20., 20.]))
np.testing.assert_array_almost_equal(rdt2['calibrated'],
np.array([22.0, 23.0]))
def test_lookup_values_ingest_replay(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsub_management.create_stream_definition('lookups', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
config = DotDict()
config.process.lookup_docs = ['test1', 'test2']
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id, config=config)
self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id)
stored_value_manager = StoredValueManager(self.container)
stored_value_manager.stored_value_cas('test1',{'offset_a':10.0, 'offset_b':13.1})
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = [20.0] * 20
granule = rdt.to_granule()
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(20))
np.testing.assert_array_almost_equal(rdt_out['temp'], np.array([20.] * 20))
np.testing.assert_array_almost_equal(rdt_out['calibrated'], np.array([30.]*20))
np.testing.assert_array_equal(rdt_out['offset_b'], np.array([rdt_out.fill_value('offset_b')] * 20))
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(20,40)
rdt['temp'] = [20.0] * 20
granule = rdt.to_granule()
dataset_monitor.event.clear()
stored_value_manager.stored_value_cas('test1',{'offset_a':20.0})
stored_value_manager.stored_value_cas('coefficient_document',{'offset_b':10.0})
gevent.sleep(2)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(40))
np.testing.assert_array_almost_equal(rdt_out['temp'], np.array([20.] * 20 + [20.] * 20))
np.testing.assert_array_equal(rdt_out['offset_b'], np.array([10.] * 40))
np.testing.assert_array_almost_equal(rdt_out['calibrated'], np.array([30.]*20 + [40.]*20))
np.testing.assert_array_almost_equal(rdt_out['calibrated_b'], np.array([40.] * 20 + [50.] * 20))
def populate_qc_tables(self):
svm = StoredValueManager(self.container)
svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':-2., 'grt_max_value':40.})
svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4})
svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc': 0.1, 'spike_n':5, 'spike_l':5})
class TestQCFunctions(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
self.pdict_id = self.ph.create_simple_qc_pdict()
self.svm = StoredValueManager(self.container)
def new_rdt(self,ref='QCTEST'):
self.stream_def_id = self.create_stream_definition(uuid4().hex, parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
def test_qc_functions(self):
self.check_global_range()
self.check_spike()
self.check_stuck_value()
self.check_trend()
self.check_gradient()
self.check_localrange()
self.check_propagate()
def check_global_range(self):
log.info('check_global_range')
self.new_rdt()
self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_spike(self):
log.info('check_spike')
self.new_rdt()
self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1])
def check_stuck_value(self):
log.info('check_stuck_value')
self.new_rdt()
self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
def check_trend(self):
log.info('check_trend')
self.new_rdt()
self.svm.stored_value_cas('trend_QCTEST_TEMPWAT', {'time_interval':0, 'polynomial_order': 1, 'standard_deviation': 3})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [0.8147, 0.9058, 0.1270, 0.9134, 0.6324, 0.0975, 0.2785, 0.5469, 0.9575, 0.9649]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_trndtst_qc'], [1] * 10)
def check_propagate(self):
log.info('check_propagate')
self.new_rdt()
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt['tempwat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
np.testing.assert_array_equal(self.rdt['cmbnflg_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_gradient(self):
log.info('check_gradient')
self.new_rdt()
self.svm.stored_value_cas('grad_QCTEST_TEMPWAT_time', {'d_dat_dx': 50, 'min_dx': 0, 'start_dat': 0, 'tol_dat': 5})
self.rdt['time'] = np.arange(5)
self.rdt['temp'] = [3, 5, 98, 99, 4]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_gradtst_qc'], [1, 1, 0, 0, 1])
def check_localrange(self):
log.info('check_localrange')
self.new_rdt()
t = np.array([3580144703.7555027, 3580144704.7555027, 3580144705.7555027, 3580144706.7555027, 3580144707.7555027, 3580144708.7555027, 3580144709.7555027, 3580144710.7555027, 3580144711.7555027, 3580144712.7555027])
pressure = np.random.rand(10) * 2 + 33.0
t_v = ntp_to_month(t)
dat = t_v + pressure + np.arange(16,26)
def lim1(p,m):
return p+m+10
def lim2(p,m):
return p+m+20
pressure_grid, month_grid = np.meshgrid(np.arange(0,150,10), np.arange(11))
points = np.column_stack([pressure_grid.flatten(), month_grid.flatten()])
datlim_0 = lim1(points[:,0], points[:,1])
datlim_1 = lim2(points[:,0], points[:,1])
datlim = np.column_stack([datlim_0, datlim_1])
datlimz = points
self.svm.stored_value_cas('lrt_QCTEST_TEMPWAT', {'datlim':datlim.tolist(), 'datlimz':datlimz.tolist(), 'dims':['pressure', 'month']})
self.rdt['time'] = t
self.rdt['temp'] = dat
self.rdt['pressure'] = pressure
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1 ,1 ,1 ,1 ,1 ,0 ,0 ,0 ,0 ,0])
def test_lookup_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsubcli.create_stream_definition('lookup', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id)
data_product = DataProduct(name='lookup data product')
tdom, sdom = time_series_domain()
data_product.temporal_domain = tdom.dump()
data_product.spatial_domain = sdom.dump()
data_product_id = self.dpsc_cli.create_data_product(data_product, stream_definition_id=stream_def_id)
self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id)
data_producer = DataProducer(name='producer')
data_producer.producer_context = DataProcessProducerContext()
data_producer.producer_context.configuration['qc_keys'] = ['offset_document']
data_producer_id, _ = self.rrclient.create(data_producer)
self.addCleanup(self.rrclient.delete, data_producer_id)
assoc,_ = self.rrclient.create_association(subject=data_product_id, object=data_producer_id, predicate=PRED.hasDataProducer)
self.addCleanup(self.rrclient.delete_association, assoc)
document_keys = self.damsclient.list_qc_references(data_product_id)
self.assertEquals(document_keys, ['offset_document'])
svm = StoredValueManager(self.container)
svm.stored_value_cas('offset_document', {'offset_a':2.0})
self.dpsc_cli.activate_data_product_persistence(data_product_id)
dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasDataset, id_only=True)
dataset_id = dataset_ids[0]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20.]
granule = rdt.to_granule()
stream_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp'], rdt2['temp'])
np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0]))
svm.stored_value_cas('updated_document', {'offset_a':3.0})
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent)
ep.publish_event(origin=data_product_id, reference_keys=['updated_document'])
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1]
rdt['temp'] = [20.]
granule = rdt.to_granule()
gevent.sleep(2) # Yield so that the event goes through
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt2['temp'],np.array([20.,20.]))
np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0,23.0]))
def test_lookup_values_ingest_replay(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsub_management.create_stream_definition(
'lookups', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
stream_id, route = self.pubsub_management.create_stream(
'example',
exchange_point=self.exchange_point_name,
stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
config = DotDict()
config.process.lookup_docs = ['test1', 'test2']
self.ingestion_management.persist_data_stream(
stream_id=stream_id,
ingestion_configuration_id=ingestion_config_id,
dataset_id=dataset_id,
config=config)
self.addCleanup(self.ingestion_management.unpersist_data_stream,
stream_id, ingestion_config_id)
stored_value_manager = StoredValueManager(self.container)
stored_value_manager.stored_value_cas('test1', {
'offset_a': 10.0,
'offset_b': 13.1
})
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = [20.0] * 20
granule = rdt.to_granule()
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(20))
np.testing.assert_array_almost_equal(rdt_out['temp'],
np.array([20.] * 20))
np.testing.assert_array_almost_equal(rdt_out['calibrated'],
np.array([30.] * 20))
np.testing.assert_array_equal(
rdt_out['offset_b'],
np.array([rdt_out.fill_value('offset_b')] * 20))
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(20, 40)
rdt['temp'] = [20.0] * 20
granule = rdt.to_granule()
dataset_monitor.event.clear()
stored_value_manager.stored_value_cas('test1', {'offset_a': 20.0})
stored_value_manager.stored_value_cas('coefficient_document',
{'offset_b': 10.0})
gevent.sleep(2)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(30))
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(40))
np.testing.assert_array_almost_equal(rdt_out['temp'],
np.array([20.] * 20 + [20.] * 20))
np.testing.assert_array_equal(rdt_out['offset_b'],
np.array([10.] * 40))
np.testing.assert_array_almost_equal(rdt_out['calibrated'],
np.array([30.] * 20 + [40.] * 20))
np.testing.assert_array_almost_equal(rdt_out['calibrated_b'],
np.array([40.] * 20 + [50.] * 20))
class TestQCFunctions(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
self.pdict_id = self.ph.create_simple_qc_pdict()
self.svm = StoredValueManager(self.container)
def new_rdt(self,ref='QCTEST'):
self.stream_def_id = self.create_stream_definition(uuid4().hex, parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
def test_qc_functions(self):
self.check_global_range()
self.check_spike()
self.check_stuck_value()
self.check_trend()
self.check_gradient()
self.check_localrange()
self.check_propagate()
def check_global_range(self):
log.info('check_global_range')
self.new_rdt()
self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_spike(self):
log.info('check_spike')
self.new_rdt()
self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1])
def check_stuck_value(self):
log.info('check_stuck_value')
self.new_rdt()
self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
def check_trend(self):
log.info('check_trend')
self.new_rdt()
self.svm.stored_value_cas('trend_QCTEST_TEMPWAT', {'time_interval':0, 'polynomial_order': 1, 'standard_deviation': 3})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [0.8147, 0.9058, 0.1270, 0.9134, 0.6324, 0.0975, 0.2785, 0.5469, 0.9575, 0.9649]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_trndtst_qc'], [1] * 10)
def check_propagate(self):
log.info('check_propagate')
self.new_rdt()
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt['tempwat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
np.testing.assert_array_equal(self.rdt['cmbnflg_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_gradient(self):
log.info('check_gradient')
self.new_rdt()
self.svm.stored_value_cas('grad_QCTEST_TEMPWAT_time', {'d_dat_dx': 50, 'min_dx': 0, 'start_dat': 0, 'tol_dat': 5})
self.rdt['time'] = np.arange(5)
self.rdt['temp'] = [3, 5, 98, 99, 4]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_gradtst_qc'], [1, 1, 0, 0, 1])
def check_localrange(self):
log.info('check_localrange')
self.new_rdt()
t = np.array([3580144703.7555027, 3580144704.7555027, 3580144705.7555027, 3580144706.7555027, 3580144707.7555027, 3580144708.7555027, 3580144709.7555027, 3580144710.7555027, 3580144711.7555027, 3580144712.7555027])
pressure = np.random.rand(10) * 2 + 33.0
t_v = ntp_to_month(t)
dat = t_v + pressure + np.arange(16,26)
def lim1(p,m):
return p+m+10
def lim2(p,m):
return p+m+20
pressure_grid, month_grid = np.meshgrid(np.arange(0,150,10), np.arange(11))
points = np.column_stack([pressure_grid.flatten(), month_grid.flatten()])
datlim_0 = lim1(points[:,0], points[:,1])
datlim_1 = lim2(points[:,0], points[:,1])
datlim = np.column_stack([datlim_0, datlim_1])
datlimz = points
self.svm.stored_value_cas('lrt_QCTEST_TEMPWAT', {'datlim':datlim.tolist(), 'datlimz':datlimz.tolist(), 'dims':['pressure', 'month']})
self.rdt['time'] = t
self.rdt['temp'] = dat
self.rdt['pressure'] = pressure
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1 ,1 ,1 ,1 ,1 ,0 ,0 ,0 ,0 ,0])
