def test_calculate_external_single_deployment(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels', 'frame_type', 'nutnr_dark_value_used_for_fit']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() nut_stream_dataset = StreamDataset(self.nutnr_sk, {}, [self.ctdpf_sk], 'UNIT') nut_stream_dataset.events = self.nut_events nut_stream_dataset._insert_dataset(nut_ds) nut_stream_dataset.calculate_all() nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset}) nut_stream_dataset.calculate_all() expected_params = ['ctdpf_sbe43_sample-seawater_temperature', 'ctdpf_sbe43_sample-practical_salinity', 'salinity_corrected_nitrate'] self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def test_calculate_external_multiple_deployments(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels', 'frame_type', 'nutnr_dark_value_used_for_fit']] # remap times to make this two separate deployments dep1_start = self.ctd_events.deps[1].ntp_start dep2_stop = self.ctd_events.deps[2].ntp_start + 864000 ctd_ds.time.values = np.linspace(dep1_start + 1, dep2_stop - 1, num=ctd_ds.time.shape[0]) nut_ds.time.values = np.linspace(dep1_start + 1, dep2_stop - 1, num=nut_ds.time.shape[0]) ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() nut_stream_dataset = StreamDataset(self.nutnr_sk, {}, [self.ctdpf_sk], 'UNIT') nut_stream_dataset.events = self.nut_events nut_stream_dataset._insert_dataset(nut_ds) nut_stream_dataset.calculate_all() nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset}) nut_stream_dataset.calculate_all() expected_params = ['ctdpf_sbe43_sample-seawater_temperature', 'ctdpf_sbe43_sample-practical_salinity', 'salinity_corrected_nitrate'] self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def test_calculate_external_multiple_deployments(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels', 'frame_type', 'nutnr_dark_value_used_for_fit']] # remap times to make this two separate deployments dep1_start = self.ctd_events.deps[1].ntp_start dep2_stop = self.ctd_events.deps[2].ntp_start + 864000 ctd_ds.time.values = np.linspace(dep1_start + 1, dep2_stop - 1, num=ctd_ds.time.shape[0]) nut_ds.time.values = np.linspace(dep1_start + 1, dep2_stop - 1, num=nut_ds.time.shape[0]) ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() nut_stream_dataset = StreamDataset(self.nutnr_sk, {}, [self.ctdpf_sk], 'UNIT') nut_stream_dataset.events = self.nut_events nut_stream_dataset._insert_dataset(nut_ds) nut_stream_dataset.calculate_all() nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset}) nut_stream_dataset.calculate_all() expected_params = ['ctdpf_sbe43_sample-seawater_temperature', 'ctdpf_sbe43_sample-practical_salinity', 'salinity_corrected_nitrate'] self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def test_calculate_external_single_deployment(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) nut_ds = xr.open_dataset(os.path.join(DATA_DIR, self.nutnr_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] nut_ds = nut_ds[['obs', 'time', 'deployment', 'spectral_channels', 'frame_type', 'nutnr_dark_value_used_for_fit']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() nut_stream_dataset = StreamDataset(self.nutnr_sk, {}, [self.ctdpf_sk], 'UNIT') nut_stream_dataset.events = self.nut_events nut_stream_dataset._insert_dataset(nut_ds) nut_stream_dataset.calculate_all() nut_stream_dataset.interpolate_needed({self.ctdpf_sk: ctd_stream_dataset}) nut_stream_dataset.calculate_all() expected_params = ['ctdpf_sbe43_sample-seawater_temperature', 'ctdpf_sbe43_sample-practical_salinity', 'salinity_corrected_nitrate'] self.assert_parameters_in_datasets(nut_stream_dataset.datasets, expected_params)
def test_calculate_external_12035(self): velpt_ds = xr.open_dataset(os.path.join(DATA_DIR, self.velpt_fn), decode_times=False) metbk_ds = xr.open_dataset(os.path.join(DATA_DIR, self.metbk_fn), decode_times=False) velpt_ds = velpt_ds[[ 'obs', 'time', 'deployment', 'velocity_beam1', 'velocity_beam2', 'velocity_beam3', 'amplitude_beam1', 'amplitude_beam2', 'amplitude_beam3' ]] metbk_ds = metbk_ds[[ 'obs', 'time', 'deployment', 'barometric_pressure', 'relative_humidity', 'air_temperature', 'longwave_irradiance', 'precipitation', 'sea_surface_temperature', 'sea_surface_conductivity', 'shortwave_irradiance', 'eastward_wind_velocity', 'northward_wind_velocity' ]] velpt_stream_dataset = StreamDataset(self.velpt_sk, {}, [], 'UNIT') velpt_stream_dataset.events = self.velpt_events velpt_stream_dataset._insert_dataset(velpt_ds) velpt_stream_dataset.calculate_all() metbk_stream_dataset = StreamDataset(self.metbk_sk, {}, [self.velpt_sk], 'UNIT') metbk_stream_dataset.events = self.metbk_events metbk_stream_dataset._insert_dataset(metbk_ds) metbk_stream_dataset.calculate_all() metbk_stream_dataset.interpolate_needed( {self.velpt_sk: velpt_stream_dataset}) metbk_stream_dataset.calculate_all() expected_params = [ 'met_barpres', 'met_windavg_mag_corr_east', 'met_windavg_mag_corr_north', 'met_current_direction', 'met_current_speed', 'met_relwind_direction', 'met_relwind_speed', 'met_netsirr', 'met_salsurf', 'met_spechum', 'met_heatflx_minute', 'met_latnflx_minute', 'met_netlirr_minute', 'met_sensflx_minute', ] self.assert_parameters_in_datasets(metbk_stream_dataset.datasets, expected_params)
def test_calculate_internal_multiple_deployments(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[[ 'obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0' ]] # remap times to make this two separate deployments dep1_start = self.ctd_events.deps[1].ntp_start dep2_stop = self.ctd_events.deps[2].ntp_start + 864000 ctd_ds.time.values = np.linspace(dep1_start + 1, dep2_stop - 1, num=ctd_ds.time.shape[0]) ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() for deployment in ctd_stream_dataset.datasets: ds = ctd_stream_dataset.datasets[deployment] tempwat = ctd_sbe16plus_tempwat( ds.temperature, ctd_stream_dataset.events.get_cal('CC_a0', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a1', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a2', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a3', deployment)[0][2]) np.testing.assert_array_equal(ds.seawater_temperature, tempwat) pracsal = ctd_pracsal(ds.seawater_conductivity, ds.seawater_temperature, ds.seawater_pressure) np.testing.assert_array_equal(ds.practical_salinity, pracsal)
def test_calculate_internal_single_deployment(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[[ 'obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0' ]] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() for deployment in ctd_stream_dataset.datasets: ds = ctd_stream_dataset.datasets[deployment] tempwat = ctd_sbe16plus_tempwat( ds.temperature, ctd_stream_dataset.events.get_cal('CC_a0', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a1', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a2', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a3', deployment)[0][2]) np.testing.assert_array_equal(ds.seawater_temperature, tempwat) pracsal = ctd_pracsal(ds.seawater_conductivity, ds.seawater_temperature, ds.seawater_pressure) np.testing.assert_array_equal(ds.practical_salinity, pracsal)
def test_exclude_data(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] times = ctd_ds.time.values ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.exclude_flagged_data() np.testing.assert_array_equal(times, ctd_stream_dataset.datasets[2].time.values) # exclude a bit start = ntplib.ntp_to_system_time(times[0]) * 1000 stop = ntplib.ntp_to_system_time(times[100]) * 1000 anno = self._create_exclusion_anno(start, stop) ctd_stream_dataset.annotation_store.add_annotations([anno]) ctd_stream_dataset.exclude_flagged_data() np.testing.assert_array_equal(times[101:], ctd_stream_dataset.datasets[2].time.values) # exclude everything start = ntplib.ntp_to_system_time(times[0]) * 1000 stop = ntplib.ntp_to_system_time(times[-1]) * 1000 anno = self._create_exclusion_anno(start, stop) ctd_stream_dataset.annotation_store.add_annotations([anno]) ctd_stream_dataset.exclude_flagged_data() self.assertNotIn(2, ctd_stream_dataset.datasets)
def test_exclude_data(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] times = ctd_ds.time.values store = AnnotationStore() ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.exclude_flagged_data(store) np.testing.assert_array_equal(times, ctd_stream_dataset.datasets[2].time.values) # exclude a bit start = ntplib.ntp_to_system_time(times[0]) * 1000 stop = ntplib.ntp_to_system_time(times[100]) * 1000 anno = self._create_exclusion_anno(self.ctdpf_sk, start, stop) store.add_annotations([anno]) ctd_stream_dataset.exclude_flagged_data(store) np.testing.assert_array_equal(times[101:], ctd_stream_dataset.datasets[2].time.values) # exclude everything start = ntplib.ntp_to_system_time(times[0]) * 1000 stop = ntplib.ntp_to_system_time(times[-1]) * 1000 anno = self._create_exclusion_anno(self.ctdpf_sk, start, stop) store.add_annotations([anno]) ctd_stream_dataset.exclude_flagged_data(store) self.assertNotIn(2, ctd_stream_dataset.datasets)
def test_calculate_internal_multiple_deployments(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] # remap times to make this two separate deployments dep1_start = self.ctd_events.deps[1].ntp_start dep2_stop = self.ctd_events.deps[2].ntp_start + 864000 ctd_ds.time.values = np.linspace(dep1_start+1, dep2_stop-1, num=ctd_ds.time.shape[0]) ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() for deployment in ctd_stream_dataset.datasets: ds = ctd_stream_dataset.datasets[deployment] tempwat = ctd_sbe16plus_tempwat(ds.temperature, ctd_stream_dataset.events.get_cal('CC_a0', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a1', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a2', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a3', deployment)[0][2]) np.testing.assert_array_equal(ds.seawater_temperature, tempwat) pracsal = ctd_pracsal(ds.seawater_conductivity, ds.seawater_temperature, ds.seawater_pressure) np.testing.assert_array_equal(ds.practical_salinity, pracsal)
def test_provenance_as_netcdf_attribute_missing(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = copy.deepcopy(self.ctd_events) ctd_stream_dataset.events.deps = {} ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.insert_instrument_attributes()
def test_provenance_as_netcdf_attribute_missing(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = copy.deepcopy(self.ctd_events) ctd_stream_dataset.events.deps = {} ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.insert_instrument_attributes()
def test_calculate_external_12035(self): velpt_ds = xr.open_dataset(os.path.join(DATA_DIR, self.velpt_fn), decode_times=False) metbk_ds = xr.open_dataset(os.path.join(DATA_DIR, self.metbk_fn), decode_times=False) velpt_ds = velpt_ds[['obs', 'time', 'deployment', 'velocity_beam1', 'velocity_beam2', 'velocity_beam3', 'amplitude_beam1', 'amplitude_beam2', 'amplitude_beam3']] metbk_ds = metbk_ds[['obs', 'time', 'deployment', 'barometric_pressure', 'relative_humidity', 'air_temperature', 'longwave_irradiance', 'precipitation', 'sea_surface_temperature', 'sea_surface_conductivity', 'shortwave_irradiance', 'eastward_wind_velocity', 'northward_wind_velocity']] velpt_stream_dataset = StreamDataset(self.velpt_sk, {}, [], 'UNIT') velpt_stream_dataset.events = self.velpt_events velpt_stream_dataset._insert_dataset(velpt_ds) velpt_stream_dataset.calculate_all() metbk_stream_dataset = StreamDataset(self.metbk_sk, {}, [self.velpt_sk], 'UNIT') metbk_stream_dataset.events = self.metbk_events metbk_stream_dataset._insert_dataset(metbk_ds) metbk_stream_dataset.calculate_all() metbk_stream_dataset.interpolate_needed({self.velpt_sk: velpt_stream_dataset}) metbk_stream_dataset.calculate_all() expected_params = ['met_barpres', 'met_windavg_mag_corr_east', 'met_windavg_mag_corr_north', 'met_current_direction', 'met_current_speed', 'met_relwind_direction', 'met_relwind_speed', 'met_netsirr', 'met_salsurf', 'met_spechum', 'met_heatflx_minute', 'met_latnflx_minute', 'met_netlirr_minute', 'met_sensflx_minute', ] self.assert_parameters_in_datasets(metbk_stream_dataset.datasets, expected_params)
def test_fill_missing(self): velpt_ds = xr.open_dataset(os.path.join(DATA_DIR, self.velpt_fn), decode_times=False) velpt_ds = velpt_ds[['obs', 'time', 'deployment', 'velocity_beam1', 'velocity_beam2', 'velocity_beam3', 'amplitude_beam1', 'amplitude_beam2', 'amplitude_beam3']] velpt_stream_dataset = StreamDataset(self.velpt_sk, {}, [], 'UNIT') velpt_stream_dataset.events = self.velpt_events velpt_stream_dataset._insert_dataset(velpt_ds) velpt_stream_dataset.fill_missing() expected_params = ['eastward_velocity'] self.assert_parameters_in_datasets(velpt_stream_dataset.datasets, expected_params, expect_fill=True)
def test_fill_missing(self): velpt_ds = xr.open_dataset(os.path.join(DATA_DIR, self.velpt_fn), decode_times=False) velpt_ds = velpt_ds[['obs', 'time', 'deployment', 'velocity_beam1', 'velocity_beam2', 'velocity_beam3', 'amplitude_beam1', 'amplitude_beam2', 'amplitude_beam3']] velpt_stream_dataset = StreamDataset(self.velpt_sk, {}, [], 'UNIT') velpt_stream_dataset.events = self.velpt_events velpt_stream_dataset._insert_dataset(velpt_ds) velpt_stream_dataset.fill_missing() expected_params = ['eastward_velocity'] self.assert_parameters_in_datasets(velpt_stream_dataset.datasets, expected_params, expect_fill=True)
def test_log_algorithm_inputs_no_result(self): def mock_write(self): return json.dumps(self.m_qdata, default=jdefault) uflags = {'advancedStreamEngineLogging': True, 'userName': '******'} ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, uflags, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) parameter = Parameter.query.get(13) with mock.patch('util.stream_dataset.ParameterReport.write', new=mock_write): result = ctd_stream_dataset._log_algorithm_inputs(parameter, {}, None, self.ctdpf_sk, ctd_ds) self.assertIsNotNone(result)
def test_log_algorithm_inputs_no_result(self): def mock_write(self): return json.dumps(self.m_qdata, default=jdefault) uflags = {'advancedStreamEngineLogging': True, 'userName': '******'} ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, uflags, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) parameter = Parameter.query.get(13) with mock.patch('util.stream_dataset.ParameterReport.write', new=mock_write): result = ctd_stream_dataset._log_algorithm_inputs(parameter, {}, None, self.ctdpf_sk, ctd_ds) self.assertIsNotNone(result)
def test_provenance_as_netcdf_attribute(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[[ 'obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0' ]] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.insert_instrument_attributes() for ds in ctd_stream_dataset.datasets.itervalues(): self.assertIn('Manufacturer', ds.attrs) self.assertIn('ModelNumber', ds.attrs) self.assertIn('SerialNumber', ds.attrs) self.assertIn('Description', ds.attrs) self.assertIn('FirmwareVersion', ds.attrs) self.assertIn('SoftwareVersion', ds.attrs) self.assertIn('AssetUniqueID', ds.attrs) self.assertIn('Notes', ds.attrs) self.assertIn('Owner', ds.attrs) self.assertIn('RemoteResources', ds.attrs) self.assertIn('ShelfLifeExpirationDate', ds.attrs) self.assertIn('Mobile', ds.attrs) self.assertIn('AssetManagementRecordLastModified', ds.attrs) self.assertEqual(ds.attrs['Manufacturer'], 'Sea-Bird Electronics') self.assertEqual(ds.attrs['ModelNumber'], 'SBE 16plus V2') self.assertEqual(ds.attrs['SerialNumber'], '16-50112') self.assertEqual(ds.attrs['Description'], 'CTD Profiler: CTDPF Series A') self.assertEqual(ds.attrs['AssetUniqueID'], 'ATOSU-66662-00013') self.assertEqual(ds.attrs['Mobile'], 'False') self.assertEqual(ds.attrs['AssetManagementRecordLastModified'], '2017-04-03T23:48:25.650000') self.assertEqual(ds.attrs['FirmwareVersion'], 'Not specified.') self.assertEqual(ds.attrs['SoftwareVersion'], 'Not specified.') self.assertEqual(ds.attrs['Notes'], 'Not specified.') self.assertEqual(ds.attrs['Owner'], 'Not specified.') self.assertEqual(ds.attrs['RemoteResources'], '[]') self.assertEqual(ds.attrs['ShelfLifeExpirationDate'], 'Not specified.')
def test_provenance_as_netcdf_attribute(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.insert_instrument_attributes() for ds in ctd_stream_dataset.datasets.itervalues(): self.assertIn('Manufacturer', ds.attrs) self.assertIn('ModelNumber', ds.attrs) self.assertIn('SerialNumber', ds.attrs) self.assertIn('Description', ds.attrs) self.assertIn('FirmwareVersion', ds.attrs) self.assertIn('SoftwareVersion', ds.attrs) self.assertIn('AssetUniqueID', ds.attrs) self.assertIn('Notes', ds.attrs) self.assertIn('Owner', ds.attrs) self.assertIn('RemoteResources', ds.attrs) self.assertIn('ShelfLifeExpirationDate', ds.attrs) self.assertIn('Mobile', ds.attrs) self.assertIn('AssetManagementRecordLastModified', ds.attrs) self.assertEqual(ds.attrs['Manufacturer'], 'Sea-Bird Electronics') self.assertEqual(ds.attrs['ModelNumber'], 'SBE 16plus V2') self.assertEqual(ds.attrs['SerialNumber'], '16-50112') self.assertEqual(ds.attrs['Description'], 'CTD Profiler: CTDPF Series A') self.assertEqual(ds.attrs['AssetUniqueID'], 'ATOSU-66662-00013') self.assertEqual(ds.attrs['Mobile'], 'False') self.assertEqual(ds.attrs['AssetManagementRecordLastModified'], '2017-04-03T23:48:25.650000') self.assertEqual(ds.attrs['FirmwareVersion'], 'Not specified.') self.assertEqual(ds.attrs['SoftwareVersion'], 'Not specified.') self.assertEqual(ds.attrs['Notes'], 'Not specified.') self.assertEqual(ds.attrs['Owner'], 'Not specified.') self.assertEqual(ds.attrs['RemoteResources'], '[]') self.assertEqual(ds.attrs['ShelfLifeExpirationDate'], 'Not specified.')
def test_calculate_internal_single_deployment(self): ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False) ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure', 'pressure_temp', 'conductivity', 'ext_volt0']] ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT') ctd_stream_dataset.events = self.ctd_events ctd_stream_dataset._insert_dataset(ctd_ds) ctd_stream_dataset.calculate_all() for deployment in ctd_stream_dataset.datasets: ds = ctd_stream_dataset.datasets[deployment] tempwat = ctd_sbe16plus_tempwat(ds.temperature, ctd_stream_dataset.events.get_cal('CC_a0', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a1', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a2', deployment)[0][2], ctd_stream_dataset.events.get_cal('CC_a3', deployment)[0][2]) np.testing.assert_array_equal(ds.seawater_temperature, tempwat) pracsal = ctd_pracsal(ds.seawater_conductivity, ds.seawater_temperature, ds.seawater_pressure) np.testing.assert_array_equal(ds.practical_salinity, pracsal)
def fetch_raw_data(self): """ Fetch the source data for this request :return: """ # Start fetching calibration data from Asset Management am_events = {} am_futures = {} for stream_key in self.stream_parameters: refdes = '-'.join((stream_key.subsite, stream_key.node, stream_key.sensor)) am_futures[stream_key] = self.asset_management.get_events_async(refdes) # Resolve calibration data futures and attach to instrument data for stream_key in am_futures: events = am_futures[stream_key].result() am_events[stream_key] = events # Start fetching instrument data for stream_key, stream_parameters in self.stream_parameters.iteritems(): other_streams = set(self.stream_parameters) other_streams.remove(stream_key) should_pad = stream_key != self.stream_key if not stream_key.is_virtual: log.debug('<%s> Fetching raw data for %s', self.request_id, stream_key.as_refdes()) sd = StreamDataset(stream_key, self.uflags, other_streams, self.request_id) sd.events = am_events[stream_key] try: sd.fetch_raw_data(self.time_range, self.limit, should_pad) self.datasets[stream_key] = sd except MissingDataException as e: if stream_key == self.stream_key: raise MissingDataException("Query returned no results for primary stream") elif stream_key.stream in self.stream_key.stream.source_streams: raise MissingDataException("Query returned no results for source stream") else: log.error('<%s> %s', self.request_id, e.message) else: log.debug('<%s> Creating empty dataset for virtual stream: %s', self.request_id, stream_key.as_refdes()) sd = StreamDataset(stream_key, self.uflags, other_streams, self.request_id) sd.events = am_events[stream_key] self.datasets[stream_key] = sd # Fetch annotations self._insert_annotations() self._exclude_flagged_data() self._exclude_nondeployed_data() # Verify data still exists after masking virtual message = 'Query returned no results for %s stream (due to deployment or annotation mask)' if self.stream_key.is_virtual: found_streams = [stream.stream for stream in self.datasets if self.datasets[stream]] if not any(stream in self.stream_key.stream.source_streams for stream in found_streams): raise MissingDataException(message % 'source') # real else: primary_stream_dataset = self.datasets[self.stream_key] if not primary_stream_dataset.datasets: raise MissingDataException(message % 'primary') # Remove any empty, non-virtual supporting datasets for stream_key in list(self.datasets): if not stream_key.is_virtual: if not self.datasets[stream_key].datasets: del self.datasets[stream_key]
def fetch_raw_data(self): """ Fetch the source data for this request :return: """ # Start fetching calibration data from Asset Management am_events = {} am_futures = {} for stream_key in self.stream_parameters: refdes = '-'.join( (stream_key.subsite, stream_key.node, stream_key.sensor)) am_futures[stream_key] = self.asset_management.get_events_async( refdes) # Resolve calibration data futures and attach to instrument data for stream_key in am_futures: events = am_futures[stream_key].result() am_events[stream_key] = events # Start fetching instrument data for stream_key, stream_parameters in self.stream_parameters.iteritems( ): other_streams = set(self.stream_parameters) other_streams.remove(stream_key) should_pad = stream_key != self.stream_key if not stream_key.is_virtual: log.debug('<%s> Fetching raw data for %s', self.request_id, stream_key.as_refdes()) sd = StreamDataset(stream_key, self.uflags, other_streams, self.request_id) sd.events = am_events[stream_key] try: sd.fetch_raw_data(self.time_range, self.limit, should_pad) self.datasets[stream_key] = sd except MissingDataException as e: if stream_key == self.stream_key: raise MissingDataException( "Query returned no results for primary stream") elif stream_key.stream in self.stream_key.stream.source_streams: raise MissingDataException( "Query returned no results for source stream") else: log.error('<%s> %s', self.request_id, e.message) else: log.debug('<%s> Creating empty dataset for virtual stream: %s', self.request_id, stream_key.as_refdes()) sd = StreamDataset(stream_key, self.uflags, other_streams, self.request_id) sd.events = am_events[stream_key] self.datasets[stream_key] = sd self._exclude_flagged_data() self._exclude_nondeployed_data() # Verify data still exists after masking virtual message = 'Query returned no results for %s stream (due to deployment or annotation mask)' if self.stream_key.is_virtual: found_streams = [ stream.stream for stream in self.datasets if self.datasets[stream] ] if not any(stream in self.stream_key.stream.source_streams for stream in found_streams): raise MissingDataException(message % 'source') # real else: primary_stream_dataset = self.datasets[self.stream_key] if not primary_stream_dataset.datasets: raise MissingDataException(message % 'primary') # Remove any empty, non-virtual supporting datasets for stream_key in list(self.datasets): if not stream_key.is_virtual: if not self.datasets[stream_key].datasets: del self.datasets[stream_key]