def make_data_product(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_simple_qc_pdict() stream_def_id = self.create_stream_definition('global range', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':'QCTEST'}) self.populate_qc_tables() dp_id = self.create_data_product('qc data product', stream_def_id=stream_def_id) self.activate_data_product(dp_id) dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(dp_id) return dp_id, dataset_id, stream_def_id
def make_data_product(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_simple_qc_pdict() stream_def_id = self.create_stream_definition( 'global range', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator': 'QCTEST'}) self.populate_qc_tables() dp_id = self.create_data_product('qc data product', stream_def_id=stream_def_id) self.activate_data_product(dp_id) dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset( dp_id) return dp_id, dataset_id, stream_def_id
def test_global_range_lookup(self): reference_designator = "CE01ISSM-MF005-01-CTDBPC999" ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_simple_qc_pdict() svm = StoredValueManager(self.container) doc_key = 'grt_%s_TEMPWAT' % reference_designator svm.stored_value_cas(doc_key, {'grt_min_value':-2, 'grt_max_value':40}) stream_def_id = self.pubsub_management.create_stream_definition('qc parsed', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':reference_designator}) self.addCleanup(self.pubsub_management.delete_stream_definition,stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [0] rdt['temp'] = [20] rdt.fetch_lookup_values() min_field = [i for i in rdt.fields if 'grt_min_value' in i][0] max_field = [i for i in rdt.fields if 'grt_max_value' in i][0] np.testing.assert_array_almost_equal(rdt[min_field], [-2.]) np.testing.assert_array_almost_equal(rdt[max_field], [40.]) np.testing.assert_array_almost_equal(rdt['tempwat_glblrng_qc'],[1])
class TestQCFunctions(DMTestCase): def setUp(self): DMTestCase.setUp(self) self.ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = self.ph.create_simple_qc_pdict() self.stream_def_id = self.pubsub_management.create_stream_definition('global range', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':'QCTEST'}) self.addCleanup(self.pubsub_management.delete_stream_definition, self.stream_def_id) self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) self.svm = StoredValueManager(self.container) def test_global_range_test(self): self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.}) self.rdt['time'] = np.arange(8) self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25] self.rdt.fetch_lookup_values() np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0]) def test_spike_test(self): # I know how redundant this sounds self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.}) self.rdt['time'] = np.arange(8) self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1] self.rdt.fetch_lookup_values() np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1]) def test_stuck_value_test(self): self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.}) self.rdt['time'] = np.arange(10) self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67] self.rdt.fetch_lookup_values() np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1])
class TestQCFunctions(DMTestCase): def setUp(self): DMTestCase.setUp(self) self.ph = ParameterHelper(self.dataset_management, self.addCleanup) self.pdict_id = self.ph.create_simple_qc_pdict() self.svm = StoredValueManager(self.container) def new_rdt(self,ref='QCTEST'): self.stream_def_id = self.create_stream_definition(uuid4().hex, parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':'QCTEST'}) self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) def test_qc_functions(self): self.check_global_range() self.check_spike() self.check_stuck_value() self.check_trend() self.check_gradient() self.check_localrange() self.check_propagate() def check_global_range(self): log.info('check_global_range') self.new_rdt() self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.}) self.rdt['time'] = np.arange(8) self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25] self.rdt.fetch_lookup_values() np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0]) def check_spike(self): log.info('check_spike') self.new_rdt() self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.}) self.rdt['time'] = np.arange(8) self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1] self.rdt.fetch_lookup_values() np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1]) def check_stuck_value(self): log.info('check_stuck_value') self.new_rdt() self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.}) self.rdt['time'] = np.arange(10) self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67] self.rdt.fetch_lookup_values() np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1]) def check_trend(self): log.info('check_trend') self.new_rdt() self.svm.stored_value_cas('trend_QCTEST_TEMPWAT', {'time_interval':0, 'polynomial_order': 1, 'standard_deviation': 3}) self.rdt['time'] = np.arange(10) self.rdt['temp'] = [0.8147, 0.9058, 0.1270, 0.9134, 0.6324, 0.0975, 0.2785, 0.5469, 0.9575, 0.9649] self.rdt.fetch_lookup_values() np.testing.assert_array_equal(self.rdt['tempwat_trndtst_qc'], [1] * 10) def check_propagate(self): log.info('check_propagate') self.new_rdt() self.rdt['time'] = np.arange(8) self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25] self.rdt['tempwat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] np.testing.assert_array_equal(self.rdt['cmbnflg_qc'], [0, 1, 1, 1, 1, 1, 1, 0]) def check_gradient(self): log.info('check_gradient') self.new_rdt() self.svm.stored_value_cas('grad_QCTEST_TEMPWAT_time', {'d_dat_dx': 50, 'min_dx': 0, 'start_dat': 0, 'tol_dat': 5}) self.rdt['time'] = np.arange(5) self.rdt['temp'] = [3, 5, 98, 99, 4] self.rdt.fetch_lookup_values() np.testing.assert_array_equal(self.rdt['tempwat_gradtst_qc'], [1, 1, 0, 0, 1]) def check_localrange(self): log.info('check_localrange') self.new_rdt() t = np.array([3580144703.7555027, 3580144704.7555027, 3580144705.7555027, 3580144706.7555027, 3580144707.7555027, 3580144708.7555027, 3580144709.7555027, 3580144710.7555027, 3580144711.7555027, 3580144712.7555027]) pressure = np.random.rand(10) * 2 + 33.0 t_v = ntp_to_month(t) dat = t_v + pressure + np.arange(16,26) def lim1(p,m): return p+m+10 def lim2(p,m): return p+m+20 pressure_grid, month_grid = np.meshgrid(np.arange(0,150,10), np.arange(11)) points = np.column_stack([pressure_grid.flatten(), month_grid.flatten()]) datlim_0 = lim1(points[:,0], points[:,1]) datlim_1 = lim2(points[:,0], points[:,1]) datlim = np.column_stack([datlim_0, datlim_1]) datlimz = points self.svm.stored_value_cas('lrt_QCTEST_TEMPWAT', {'datlim':datlim.tolist(), 'datlimz':datlimz.tolist(), 'dims':['pressure', 'month']}) self.rdt['time'] = t self.rdt['temp'] = dat self.rdt['pressure'] = pressure self.rdt.fetch_lookup_values() np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1 ,1 ,1 ,1 ,1 ,0 ,0 ,0 ,0 ,0])
class TestQCFunctions(DMTestCase): def setUp(self): DMTestCase.setUp(self) self.ph = ParameterHelper(self.dataset_management, self.addCleanup) self.pdict_id = self.ph.create_simple_qc_pdict() self.svm = StoredValueManager(self.container) def new_rdt(self,ref='QCTEST'): self.stream_def_id = self.create_stream_definition(uuid4().hex, parameter_dictionary_id=self.pdict_id, stream_configuration={'reference_designator':'QCTEST'}) self.rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) def test_qc_functions(self): self.check_global_range() self.check_spike() self.check_stuck_value() self.check_trend() self.check_gradient() self.check_localrange() self.check_propagate() def check_global_range(self): log.info('check_global_range') self.new_rdt() self.svm.stored_value_cas('grt_QCTEST_TEMPWAT', {'grt_min_value':10., 'grt_max_value':20.}) self.rdt['time'] = np.arange(8) self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25] self.rdt.fetch_lookup_values() np.testing.assert_array_almost_equal(self.rdt['tempwat_glblrng_qc'], [0, 1, 1, 1, 1, 1, 1, 0]) def check_spike(self): log.info('check_spike') self.new_rdt() self.svm.stored_value_cas('spike_QCTEST_TEMPWAT', {'acc':0.1, 'spike_n':5., 'spike_l':5.}) self.rdt['time'] = np.arange(8) self.rdt['temp'] = [-1, 3, 40, -1, 1, -6, -6, 1] self.rdt.fetch_lookup_values() np.testing.assert_array_almost_equal(self.rdt['tempwat_spketst_qc'], [1, 1, 0, 1, 1, 1, 1, 1]) def check_stuck_value(self): log.info('check_stuck_value') self.new_rdt() self.svm.stored_value_cas('svt_QCTEST_TEMPWAT', {'svt_resolution':0.001, 'svt_n': 4.}) self.rdt['time'] = np.arange(10) self.rdt['temp'] = [4.83, 1.40, 3.33, 3.33, 3.33, 3.33, 4.09, 2.97, 2.85, 3.67] self.rdt.fetch_lookup_values() np.testing.assert_array_almost_equal(self.rdt['tempwat_stuckvl_qc'], [1, 1, 0, 0, 0, 0, 1, 1, 1, 1]) def check_trend(self): log.info('check_trend') self.new_rdt() self.svm.stored_value_cas('trend_QCTEST_TEMPWAT', {'time_interval':0, 'polynomial_order': 1, 'standard_deviation': 3}) self.rdt['time'] = np.arange(10) self.rdt['temp'] = [0.8147, 0.9058, 0.1270, 0.9134, 0.6324, 0.0975, 0.2785, 0.5469, 0.9575, 0.9649] self.rdt.fetch_lookup_values() np.testing.assert_array_equal(self.rdt['tempwat_trndtst_qc'], [1] * 10) def check_propagate(self): log.info('check_propagate') self.new_rdt() self.rdt['time'] = np.arange(8) self.rdt['temp'] = [9, 10, 16, 17, 18, 19, 20, 25] self.rdt['tempwat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['tempwat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_glblrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_spketst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_stuckvl_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_gradtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_trndtst_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] self.rdt['preswat_loclrng_qc'] = [0, 1, 1, 1, 1, 1, 1, 0] np.testing.assert_array_equal(self.rdt['cmbnflg_qc'], [0, 1, 1, 1, 1, 1, 1, 0]) def check_gradient(self): log.info('check_gradient') self.new_rdt() self.svm.stored_value_cas('grad_QCTEST_TEMPWAT_time', {'d_dat_dx': 50, 'min_dx': 0, 'start_dat': 0, 'tol_dat': 5}) self.rdt['time'] = np.arange(5) self.rdt['temp'] = [3, 5, 98, 99, 4] self.rdt.fetch_lookup_values() np.testing.assert_array_equal(self.rdt['tempwat_gradtst_qc'], [1, 1, 0, 0, 1]) def check_localrange(self): log.info('check_localrange') self.new_rdt() t = np.array([3580144703.7555027, 3580144704.7555027, 3580144705.7555027, 3580144706.7555027, 3580144707.7555027, 3580144708.7555027, 3580144709.7555027, 3580144710.7555027, 3580144711.7555027, 3580144712.7555027]) pressure = np.random.rand(10) * 2 + 33.0 t_v = ntp_to_month(t) dat = t_v + pressure + np.arange(16,26) def lim1(p,m): return p+m+10 def lim2(p,m): return p+m+20 pressure_grid, month_grid = np.meshgrid(np.arange(0,150,10), np.arange(11)) points = np.column_stack([pressure_grid.flatten(), month_grid.flatten()]) datlim_0 = lim1(points[:,0], points[:,1]) datlim_1 = lim2(points[:,0], points[:,1]) datlim = np.column_stack([datlim_0, datlim_1]) datlimz = points self.svm.stored_value_cas('lrt_QCTEST_TEMPWAT', {'datlim':datlim.tolist(), 'datlimz':datlimz.tolist(), 'dims':['pressure', 'month']}) self.rdt['time'] = t self.rdt['temp'] = dat self.rdt['pressure'] = pressure self.rdt.fetch_lookup_values() np.testing.assert_array_equal(self.rdt['tempwat_loclrng_qc'], [1 ,1 ,1 ,1 ,1 ,0 ,0 ,0 ,0 ,0])