class TestQCFunctions(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = self.ph.create_simple_qc_pdict()
self.stream_def_id = self.pubsub_management.create_stream_definition('global range', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.addCleanup(self.pubsub_management.delete_stream_definition, self.stream_def_id)
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.svm = StoredValueManager(self.container)
def test_global_range_test(self):
self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def test_spike_test(self): # I know how redundant this sounds
self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1])
def test_stuck_value_test(self):
self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
class TestQCFunctions(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
self.pdict_id = self.ph.create_simple_qc_pdict()
self.svm = StoredValueManager(self.container)
def new_rdt(self,ref='QCTEST'):
self.stream_def_id = self.create_stream_definition(uuid4().hex, parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
def test_qc_functions(self):
self.check_global_range()
self.check_spike()
self.check_stuck_value()
self.check_trend()
self.check_gradient()
self.check_localrange()
self.check_propagate()
def check_global_range(self):
log.info('check_global_range')
self.new_rdt()
self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_spike(self):
log.info('check_spike')
self.new_rdt()
self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1])
def check_stuck_value(self):
log.info('check_stuck_value')
self.new_rdt()
self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
def check_trend(self):
log.info('check_trend')
self.new_rdt()
self.svm.stored_value_cas('trend_QCTEST_TEMPWAT', {'time_interval':0, 'polynomial_order': 1, 'standard_deviation': 3})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [0.8147, 0.9058, 0.1270, 0.9134, 0.6324, 0.0975, 0.2785, 0.5469, 0.9575, 0.9649]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_trndtst_qc'], [1] * 10)
def check_propagate(self):
log.info('check_propagate')
self.new_rdt()
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt['tempwat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
np.testing.assert_array_equal(self.rdt['cmbnflg_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_gradient(self):
log.info('check_gradient')
self.new_rdt()
self.svm.stored_value_cas('grad_QCTEST_TEMPWAT_time', {'d_dat_dx': 50, 'min_dx': 0, 'start_dat': 0, 'tol_dat': 5})
self.rdt['time'] = np.arange(5)
self.rdt['temp'] = [3, 5, 98, 99, 4]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_gradtst_qc'], [1, 1, 0, 0, 1])
def check_localrange(self):
log.info('check_localrange')
self.new_rdt()
t = np.array([3580144703.7555027, 3580144704.7555027, 3580144705.7555027, 3580144706.7555027, 3580144707.7555027, 3580144708.7555027, 3580144709.7555027, 3580144710.7555027, 3580144711.7555027, 3580144712.7555027])
pressure = np.random.rand(10) * 2 + 33.0
t_v = ntp_to_month(t)
dat = t_v + pressure + np.arange(16,26)
def lim1(p,m):
return p+m+10
def lim2(p,m):
return p+m+20
pressure_grid, month_grid = np.meshgrid(np.arange(0,150,10), np.arange(11))
points = np.column_stack([pressure_grid.flatten(), month_grid.flatten()])
datlim_0 = lim1(points[:,0], points[:,1])
datlim_1 = lim2(points[:,0], points[:,1])
datlim = np.column_stack([datlim_0, datlim_1])
datlimz = points
self.svm.stored_value_cas('lrt_QCTEST_TEMPWAT', {'datlim':datlim.tolist(), 'datlimz':datlimz.tolist(), 'dims':['pressure', 'month']})
self.rdt['time'] = t
self.rdt['temp'] = dat
self.rdt['pressure'] = pressure
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1 ,1 ,1 ,1 ,1 ,0 ,0 ,0 ,0 ,0])
class TestQCPreload(TestQCFunctions):
def setUp(self):
TestQCFunctions.setUp(self)
self.preload()
def preload(self):
config = DotDict()
config.op = 'load'
config.scenario='BETA'
config.categories='Parser,Reference'
config.path='master'
self.container.spawn_process('ion_loader', 'ion.processes.bootstrap.ion_loader','IONLoader',config)
def loclrng_checks(self):
ref = 'GP03FLMA-RI001-06-CTDMOG999'
self.stream_def_id = self.create_stream_definition('local range check', parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':ref})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.rdt['time'] = np.arange(10)
lon, lat, pressure = [-124.832179, 46.436926, 37.5]
self.rdt['lon'] = lon
self.rdt['lat'] = lat
self.rdt['pressure'] = [37.5] * 10
self.rdt['temp'] = [30]*10
self.rdt.fetch_lookup_values()
doc = self.svm.read_value('lrt_%s_TEMPWAT' % ref)
self.assertEquals(doc['dims'], ['lon', 'lat', 'pressure'])
doc = self.svm.read_value('lrt_%s_PRESWAT' % ref)
self.assertEquals(doc['dims'], ['temp'])
np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1]*10)
self.rdt['pressure'] = [37.5] * 9 + [75.]
self.rdt['temp'] = [30] * 9 + [18]
np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1]*9 + [0])
self.rdt['temp'] = [15] * 5 + [35] * 5
self.rdt['pressure'] = [10] * 10
np.testing.assert_array_equal(self.rdt['preswat_loclrng_qc'], [1]*5 + [0] * 5)
def glblrng_checks(self):
ref = 'CE01ISSM-MF005-01-CTDBPC999'
self.stream_def_id = self.create_stream_definition('global range check', parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':ref})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [-10] * 5 + [4] * 5
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_glblrng_qc'], [0]*5 + [1]*5)
def spketst_checks(self):
ref = 'GP02HYPM-SP001-04-CTDPF0999'
self.stream_def_id = self.create_stream_definition('spike test check', parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':ref})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.rdt['time'] = np.arange(20)
self.rdt['temp'] = [13] * 9 + [100] + [13] * 10
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_spketst_qc'], [1]*9 + [0] + [1]*10)
def stuckvl_checks(self):
ref = 'GP02HYPM-SP001-04-CTDPF0999'
self.stream_def_id = self.create_stream_definition('stuck value checks', parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':ref})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.rdt['time'] = np.arange(50)
self.rdt['temp'] = [20] * 30 + range(20)
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_stuckvl_qc'], [0]*30 + [1]*20)
def test_qc_functions(self):
self.loclrng_checks()
self.glblrng_checks()
self.spketst_checks()
self.stuckvl_checks()
class TestQCFunctions(DMTestCase):
def setUp(self):
DMTestCase.setUp(self)
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
self.pdict_id = self.ph.create_simple_qc_pdict()
self.svm = StoredValueManager(self.container)
def new_rdt(self,ref='QCTEST'):
self.stream_def_id = self.create_stream_definition(uuid4().hex, parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':'QCTEST'})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
def test_qc_functions(self):
self.check_global_range()
self.check_spike()
self.check_stuck_value()
self.check_trend()
self.check_gradient()
self.check_localrange()
self.check_propagate()
def check_global_range(self):
log.info('check_global_range')
self.new_rdt()
self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_spike(self):
log.info('check_spike')
self.new_rdt()
self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.})
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1])
def check_stuck_value(self):
log.info('check_stuck_value')
self.new_rdt()
self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67]
self.rdt.fetch_lookup_values()
np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
def check_trend(self):
log.info('check_trend')
self.new_rdt()
self.svm.stored_value_cas('trend_QCTEST_TEMPWAT', {'time_interval':0, 'polynomial_order': 1, 'standard_deviation': 3})
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [0.8147, 0.9058, 0.1270, 0.9134, 0.6324, 0.0975, 0.2785, 0.5469, 0.9575, 0.9649]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_trndtst_qc'], [1] * 10)
def check_propagate(self):
log.info('check_propagate')
self.new_rdt()
self.rdt['time'] = np.arange(8)
self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25]
self.rdt['tempwat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['tempwat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
self.rdt['preswat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0]
np.testing.assert_array_equal(self.rdt['cmbnflg_qc'], [0, 1, 1, 1, 1, 1, 1, 0])
def check_gradient(self):
log.info('check_gradient')
self.new_rdt()
self.svm.stored_value_cas('grad_QCTEST_TEMPWAT_time', {'d_dat_dx': 50, 'min_dx': 0, 'start_dat': 0, 'tol_dat': 5})
self.rdt['time'] = np.arange(5)
self.rdt['temp'] = [3, 5, 98, 99, 4]
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_gradtst_qc'], [1, 1, 0, 0, 1])
def check_localrange(self):
log.info('check_localrange')
self.new_rdt()
t = np.array([3580144703.7555027, 3580144704.7555027, 3580144705.7555027, 3580144706.7555027, 3580144707.7555027, 3580144708.7555027, 3580144709.7555027, 3580144710.7555027, 3580144711.7555027, 3580144712.7555027])
pressure = np.random.rand(10) * 2 + 33.0
t_v = ntp_to_month(t)
dat = t_v + pressure + np.arange(16,26)
def lim1(p,m):
return p+m+10
def lim2(p,m):
return p+m+20
pressure_grid, month_grid = np.meshgrid(np.arange(0,150,10), np.arange(11))
points = np.column_stack([pressure_grid.flatten(), month_grid.flatten()])
datlim_0 = lim1(points[:,0], points[:,1])
datlim_1 = lim2(points[:,0], points[:,1])
datlim = np.column_stack([datlim_0, datlim_1])
datlimz = points
self.svm.stored_value_cas('lrt_QCTEST_TEMPWAT', {'datlim':datlim.tolist(), 'datlimz':datlimz.tolist(), 'dims':['pressure', 'month']})
self.rdt['time'] = t
self.rdt['temp'] = dat
self.rdt['pressure'] = pressure
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1 ,1 ,1 ,1 ,1 ,0 ,0 ,0 ,0 ,0])
class TestQCPreload(TestQCFunctions):
def setUp(self):
TestQCFunctions.setUp(self)
self.preload()
def preload(self):
config = DotDict()
config.op = 'load'
config.scenario='BETA'
config.categories='Parser,Reference'
config.path='master'
self.container.spawn_process('ion_loader', 'ion.processes.bootstrap.ion_loader','IONLoader',config)
def loclrng_checks(self):
ref = 'GP03FLMA-RI001-06-CTDMOG999'
self.stream_def_id = self.create_stream_definition('local range check', parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':ref})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.rdt['time'] = np.arange(10)
lon, lat, pressure = [-124.832179, 46.436926, 37.5]
self.rdt['lon'] = lon
self.rdt['lat'] = lat
self.rdt['pressure'] = [37.5] * 10
self.rdt['temp'] = [30]*10
self.rdt.fetch_lookup_values()
doc = self.svm.read_value('lrt_%s_TEMPWAT' % ref)
self.assertEquals(doc['dims'], ['lon', 'lat', 'pressure'])
doc = self.svm.read_value('lrt_%s_PRESWAT' % ref)
self.assertEquals(doc['dims'], ['temp'])
np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1]*10)
self.rdt['pressure'] = [37.5] * 9 + [75.]
self.rdt['temp'] = [30] * 9 + [18]
np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1]*9 + [0])
self.rdt['temp'] = [15] * 5 + [35] * 5
self.rdt['pressure'] = [10] * 10
np.testing.assert_array_equal(self.rdt['preswat_loclrng_qc'], [1]*5 + [0] * 5)
def glblrng_checks(self):
ref = 'CE01ISSM-MF005-01-CTDBPC999'
self.stream_def_id = self.create_stream_definition('global range check', parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':ref})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.rdt['time'] = np.arange(10)
self.rdt['temp'] = [-10] * 5 + [4] * 5
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_glblrng_qc'], [0]*5 + [1]*5)
def spketst_checks(self):
ref = 'GP02HYPM-SP001-04-CTDPF0999'
self.stream_def_id = self.create_stream_definition('spike test check', parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':ref})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.rdt['time'] = np.arange(20)
self.rdt['temp'] = [13] * 9 + [100] + [13] * 10
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_spketst_qc'], [1]*9 + [0] + [1]*10)
def stuckvl_checks(self):
ref = 'GP02HYPM-SP001-04-CTDPF0999'
self.stream_def_id = self.create_stream_definition('stuck value checks', parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':ref})
self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
self.rdt['time'] = np.arange(50)
self.rdt['temp'] = [20] * 30 + range(20)
self.rdt.fetch_lookup_values()
np.testing.assert_array_equal(self.rdt['tempwat_stuckvl_qc'], [0]*30 + [1]*20)
def test_qc_functions(self):
self.loclrng_checks()
self.glblrng_checks()
self.spketst_checks()
self.stuckvl_checks()
