def dosta_ctdbp_datalogger(ds): """ Takes data from DOSTAs connected to the CTDBP, with the data recorded by the datalogger (used by some of the EA moorings) and cleans up the data set to make it more user-friendly. Primary task is renaming the alphabet soup parameter names and dropping some parameters that are of limited or no use/value. :param ds: initial dosta data set downloaded from OOI via the M2M system :return ds: cleaned up data set """ # drop some of the variables: # dcl_controller_timestamp == time, redundant so can remove # date_time_string == internal_timestamp, redundant so can remove ds = ds.drop(['dcl_controller_timestamp', 'date_time_string']) # convert the time values from a datetime64[ns] object to a floating point number with the time in seconds ds['internal_timestamp'] = ('time', dt64_epoch(ds.internal_timestamp)) ds['internal_timestamp'].attrs = dict({ 'long_name': 'Internal CTD Clock Time', 'standard_name': 'time', 'units': 'seconds since 1970-01-01 00:00:00 0:00', 'calendar': 'gregorian', 'comment': ('Comparing the instrument internal clock versus the GPS referenced sampling time will allow for ' 'calculations of the instrument clock offset and drift. Useful when working with the ' 'recovered instrument data where no external GPS referenced clock is available.' ) }) # rename some of the variables for better clarity rename = { 'dosta_ln_optode_oxygen': 'oxygen_concentration', 'dosta_ln_optode_oxygen_qc_executed': 'oxygen_concentration_qc_executed', 'dosta_ln_optode_oxygen_qc_results': 'oxygen_concentration_qc_results', 'dissolved_oxygen': 'oxygen_concentration_corrected', 'dissolved_oxygen_qc_executed': 'oxygen_concentration_corrected_qc_executed', 'dissolved_oxygen_qc_results': 'oxygen_concentration_corrected_qc_results', 'int_ctd_pressure': 'seawater_pressure', 'temp': 'seawater_temperature', } ds = ds.rename(rename) # reset variable attributes using dictionary above for v in ds.variables: if v in ATTRS.keys(): ds[v].attrs = ATTRS[v] # add original OOINet variable name as an attribute if renamed for key, value in rename.items(): ds[value].attrs['ooinet_variable_name'] = key return ds
def combine_datasets(tdata, rhdata, ridata, resample_time): """ Load and merge data from telemetered, recovered host and recovered instrument data sets. Telemetered and recovered host data represent the same source of data, just different data delivery methods. These data files are concatenated together and only unique time records are kept. The recovered instrument data is concatenated onto the telemetered/recovered host data set and then the full data set is resampled to a common time record via median averaging. The resulting merged and resampled data set is returned for further analysis. :param tdata: telemetered data as an xarray data set or None if no data available :param rhdata: recovered host data as an xarray data set or None if no data available :param ridata: recovered instrument data as xarray data set or None if no data available :param resample_time: The resampling time period in minutes :return ds: The combined and resampled data set """ # combine the telemetered and recovered host datasets, which have the same variables if tdata and rhdata: # use concat to join the datasets and then select only unique time points ds = xr.concat([tdata, rhdata], 'time') _, index = np.unique(ds['time'], return_index=True) ds = ds.isel(time=index) elif tdata and not rhdata: # telemetered data, but no recovered host data ds = tdata elif rhdata and not tdata: # recovered host data, but no telemetered data ds = rhdata else: # no telemetered or recovered host data ds = None # combine the recovered instrument data with the telemetered/recovered host data, if both exists if ds and ridata: # first, identify any variables in ds that are not available in ridata for v in ds.variables: if v not in ridata.variables: # add an empty variable of the same type and dimensions to ridata if len(ds[v].dims) == 1: ridata[v] = ds[v].broadcast_like(ridata['station']) else: ridata[v] = ds[v].broadcast_like(ridata['time']) # next, identify any variables in ridata that are not available in ds for v in ridata.variables: if v not in ds.variables: # add an empty variable of the same type and dimensions to ridata if len(ridata[v].dims) == 1: ds[v] = ridata[v].broadcast_like(ridata['station']) else: ds[v] = ridata[v].broadcast_like(ridata['time']) # finally, concat the datasets and remove any duplicate timestamps ds = xr.concat([ds, ridata], 'time') elif ds and not ridata: pass elif ridata and not ds: # no telemetered/recovered host data, just the recovered instrument data. ds = ridata else: return None # resample the dataset onto a common time record itime = '{:d}Min'.format(resample_time) gtime = '{:d}Min'.format(resample_time * 3) ds = ds.sortby('time') avg = ds.resample(time=itime, keep_attrs=True).median() avg = avg.interpolate_na(dim='time', max_gap=gtime) # reset the time record to seconds since 1970 avg['time'] = dt64_epoch(avg.time) # add the attributes back into the data set avg.attrs = ds.attrs for v in avg.variables: if v != 'time': avg[v] = avg[v].astype(ds[v].dtype) avg[v].attrs = ds[v].attrs avg.time.attrs['long_name'] = 'Time' avg.time.attrs['standard_name'] = 'time' avg.time.attrs['axis'] = 'T' avg.time.attrs['units'] = 'seconds since 1970-01-01 00:00:00 0:00' avg.time.attrs['calendar'] = 'gregorian' return avg
def phsen_streamed(ds): """ Takes PHSEN data streamed from instruments deployed by the Regional Cabled Array and cleans up the data set to make it more user-friendly. Primary task is renaming parameters and dropping some that are of limited use. Additionally, re-organize some of the variables to permit better assessments of the data. :param ds: initial PHSEN data set recorded by the data logger system and downloaded from OOI via the M2M system :return: cleaned up and reorganized data set """ # drop some of the variables: # checksum == not used # record_type == not used # record_length == not used # signal_intensity_434, part of the light measurements array, redundant so can remove # signal_intensity_578, part of the light measurements array, redundant so can remove ds = ds.reset_coords() ds = ds.drop([ 'checksum', 'record_type', 'record_length', 'signal_intensity_434', 'signal_intensity_578' ]) # convert the internal_timestamp values from a datetime64[ns] object to a floating point number with the time in # seconds, replacing the internal_timestamp with the record_time (the internal_timestamp is incorrectly set in the # NetCDF file). ds['internal_timestamp'] = ('time', dt64_epoch(ds.record_time)) ds['internal_timestamp'].attrs = dict({ 'long_name': 'Internal SAMI-pH Clock Time', 'standard_name': 'time', 'units': 'seconds since 1970-01-01 00:00:00 0:00', 'calendar': 'gregorian', 'comment': ('Comparing the instrument internal clock versus the GPS referenced sampling time will allow for ' + 'calculations of the instrument clock offset and drift. Useful when working with the ' + 'recovered instrument data where no external GPS referenced clock is available.' ) }) ds = ds.drop(['record_time']) # rename some of the variables for better clarity rename = { 'voltage_battery': 'raw_battery_voltage', 'thermistor_start': 'raw_thermistor_start', 'thermistor_end': 'raw_thermistor_end', 'phsen_thermistor_temperature': 'thermistor_temperature', 'phsen_battery_volts': 'battery_voltage', 'ph_seawater': 'seawater_ph', 'ph_seawater_qc_executed': 'seawater_ph_qc_executed', 'ph_seawater_qc_results': 'seawater_ph_qc_results' } ds = ds.rename(rename) # now we need to reset the light and reference arrays to named variables that will be more meaningful and useful in # the final data files nrec = len(ds['time'].values) light = np.array(np.vstack(ds['ph_light_measurements'].values), dtype='int32') light = np.atleast_3d(light) light = np.reshape(light, (nrec, 23, 4)) # 4 sets of 23 seawater measurements reference_434 = light[:, :, 0] # reference signal, 434 nm signal_434 = light[:, :, 1] # signal intensity, 434 nm (PH434SI_L0) reference_578 = light[:, :, 2] # reference signal, 578 nm signal_578 = light[:, :, 3] # signal intensity, 578 nm (PH578SI_L0) refnc = np.array(np.vstack(ds['reference_light_measurements'].values), dtype='int32') refnc = np.atleast_3d(refnc) refnc = np.reshape( refnc, (nrec, 4, 4)) # 4 sets of 4 DI water measurements (blanks) blank_refrnc_434 = refnc[:, :, 0] # DI blank reference, 434 nm blank_signal_434 = refnc[:, :, 1] # DI blank signal, 434 nm blank_refrnc_578 = refnc[:, :, 2] # DI blank reference, 578 nm blank_signal_578 = refnc[:, :, 3] # DI blank signal, 578 nm # create a data set with the reference and light measurements ph = xr.Dataset( { 'blank_refrnc_434': (['time', 'blanks'], blank_refrnc_434.astype('int32')), 'blank_signal_434': (['time', 'blanks'], blank_signal_434.astype('int32')), 'blank_refrnc_578': (['time', 'blanks'], blank_refrnc_578.astype('int32')), 'blank_signal_578': (['time', 'blanks'], blank_signal_578.astype('int32')), 'reference_434': (['time', 'measurements'], reference_434.astype('int32')), 'signal_434': (['time', 'measurements'], signal_434.astype('int32')), 'reference_578': (['time', 'measurements'], reference_578.astype('int32')), 'signal_578': (['time', 'measurements'], signal_578.astype('int32')) }, coords={ 'time': ds['time'], 'measurements': np.arange(0, 23).astype('int32'), 'blanks': np.arange(0, 4).astype('int32') }) ds = ds.drop([ 'ph_light_measurements', 'reference_light_measurements', 'ph_light_measurements_dim_0', 'reference_light_measurements_dim_0' ]) # merge the data sets back together ds = ds.merge(ph) # reset some of the variable attributes, and ... for v in ds.variables: # variable level attributes if v in PHSEN: ds[v].attrs = PHSEN[v] # ... add the renamed information for key, value in rename.items(): ds[value].attrs['ooinet_variable_name'] = key # and reset some of the data types data_types = [ 'deployment', 'raw_thermistor_end', 'raw_thermistor_start', 'unique_id', 'raw_battery_voltage' ] for v in data_types: ds[v] = ds[v].astype('int32') return ds
def phsen_datalogger(ds): """ Takes PHSEN data recorded by the data loggers used in the CGSN/EA moorings and cleans up the data set to make it more user-friendly. Primary task is renaming parameters and dropping some that are of limited use. Additionally, re-organize some of the variables to permit better assessments of the data. :param ds: initial PHSEN data set recorded by the data logger system and downloaded from OOI via the M2M system :return: cleaned up and reorganized data set """ # drop some of the variables: # passed_checksum == not used # record_type == not used # record_time == not used # dcl_controller_timestamp == time, redundant so can remove # phsen_abcdef_signal_intensity_434, part of the light measurements array, redundant so can remove # phsen_abcdef_signal_intensity_578, part of the light measurements array, redundant so can remove ds = ds.reset_coords() ds = ds.drop([ 'passed_checksum', 'record_type', 'record_time', 'phsen_abcdef_signal_intensity_434', 'phsen_abcdef_signal_intensity_578', 'dcl_controller_timestamp' ]) # convert the time values from a datetime64[ns] object to a floating point number with the time in seconds ds['internal_timestamp'] = ('time', dt64_epoch(ds.internal_timestamp)) ds['internal_timestamp'].attrs = dict({ 'long_name': 'Internal SAMI-pH Clock Time', 'standard_name': 'time', 'units': 'seconds since 1970-01-01 00:00:00 0:00', 'calendar': 'gregorian', 'comment': ('Comparing the instrument internal clock versus the GPS referenced sampling time will allow for ' + 'calculations of the instrument clock offset and drift. Useful when working with the ' + 'recovered instrument data where no external GPS referenced clock is available.' ) }) # rename some of the variables for better clarity rename = { 'voltage_battery': 'raw_battery_voltage', 'thermistor_start': 'raw_thermistor_start', 'thermistor_end': 'raw_thermistor_end', 'phsen_thermistor_temperature': 'thermistor_temperature', 'phsen_abcdef_ph_seawater': 'seawater_ph', 'phsen_abcdef_ph_seawater_qc_executed': 'seawater_ph_qc_executed', 'phsen_abcdef_ph_seawater_qc_results': 'seawater_ph_qc_results' } ds = ds.rename(rename) # now we need to reset the light and reference arrays to named variables that will be more meaningful and useful in # the final data files nrec = len(ds['time'].values) light = np.array(np.vstack(ds['light_measurements'].values), dtype='int32') light = np.atleast_3d(light) light = np.reshape(light, (nrec, 23, 4)) # 4 sets of 23 seawater measurements reference_434 = light[:, :, 0] # reference signal, 434 nm signal_434 = light[:, :, 1] # signal intensity, 434 nm (PH434SI_L0) reference_578 = light[:, :, 2] # reference signal, 578 nm signal_578 = light[:, :, 3] # signal intensity, 578 nm (PH578SI_L0) refnc = np.array(np.vstack(ds['reference_light_measurements'].values), dtype='int32') refnc = np.atleast_3d(refnc) refnc = np.reshape( refnc, (nrec, 4, 4)) # 4 sets of 4 DI water measurements (blanks) blank_refrnc_434 = refnc[:, :, 0] # DI blank reference, 434 nm blank_signal_434 = refnc[:, :, 1] # DI blank signal, 434 nm blank_refrnc_578 = refnc[:, :, 2] # DI blank reference, 578 nm blank_signal_578 = refnc[:, :, 3] # DI blank signal, 578 nm # create a data set with the reference and light measurements ph = xr.Dataset( { 'blank_refrnc_434': (['time', 'blanks'], blank_refrnc_434.astype('int32')), 'blank_signal_434': (['time', 'blanks'], blank_signal_434.astype('int32')), 'blank_refrnc_578': (['time', 'blanks'], blank_refrnc_578.astype('int32')), 'blank_signal_578': (['time', 'blanks'], blank_signal_578.astype('int32')), 'reference_434': (['time', 'measurements'], reference_434.astype('int32')), 'signal_434': (['time', 'measurements'], signal_434.astype('int32')), 'reference_578': (['time', 'measurements'], reference_578.astype('int32')), 'signal_578': (['time', 'measurements'], signal_578.astype('int32')) }, coords={ 'time': ds['time'], 'measurements': np.arange(0, 23).astype('int32'), 'blanks': np.arange(0, 4).astype('int32') }) ds = ds.drop([ 'reference_light_measurements_dim_0', 'spectrum', 'light_measurements', 'reference_light_measurements' ]) # these two dimensional variables may or may not be present depending on how the data was requested. # remove them if they do exist so we can merge different data sets together maybe = [ 'phsen_abcdef_signal_intensity_434_dim_0', 'phsen_abcdef_signal_intensity_578_dim_0' ] for k, v in ds.dims.items(): if k in maybe: ds = ds.drop(k) # merge the data sets back together ds = ds.merge(ph) # test data quality ds['seawater_ph_quality_flag'] = quality_checks(ds) # reset some attributes for key, value in ATTRS.items(): for atk, atv in value.items(): if key in ds.variables: ds[key].attrs[atk] = atv # add the original variable name as an attribute, if renamed for key, value in rename.items(): ds[value].attrs['ooinet_variable_name'] = key # and reset some of the data types data_types = [ 'deployment', 'raw_thermistor_end', 'raw_thermistor_start', 'unique_id', 'raw_battery_voltage' ] for v in data_types: ds[v] = ds[v].astype('int32') return ds
def pco2a_datalogger(ds, burst=False): """ Takes pco2a data recorded by the data loggers used in the CGSN/EA moorings and cleans up the data set to make it more user-friendly. Primary task is renaming parameters and dropping some that are of limited use. Additionally, re-organize some of the variables to permit better assessments of the data. :param ds: initial pco2a data set for the air measurements downloaded from OOI via the M2M system :param burst: resample the data to an hourly, burst averaged time interval :return ds: cleaned up data set """ # drop some of the variables: # ### OOI generated parameters # date_time_string == internal_timestamp, redundant so can remove # dcl_controller_timestamp == time, redundant so can remove # supply_voltage == not used # ### Data products from upstream processing used to calculate the normalized 10 m wind, but are not needed # eastward_velocity # northward_velocity # air_temperature # met_relwind_speed # longwave_irradiance # shortwave_irradiance # relative_humidity # barometric_pressure # precipitation shared = ['date_time_string', 'dcl_controller_timestamp'] if 'supply_voltage' in ds.variables: # ... because it is in the telemetered, but not the recovered_host ... shared.append('supply_voltage') ds = ds.drop(shared) # determine if the upstream parameters are present. delete them if needed, otherwise add the required ones to make # sure the NetCDF files are consistent upstream = [ 'eastward_velocity', 'northward_velocity', 'air_temperature', 'met_relwind_speed', 'longwave_irradiance', 'shortwave_irradiance', 'relative_humidity', 'barometric_pressure', 'precipitation' ] if 'eastward_velocity' in ds.variables: ds = ds.drop(upstream) else: # METBK data was missing, add variables below to keep data sets consistent ds['sea_surface_temperature'] = ('time', ds['deployment'] * np.nan) ds['sea_surface_temperature'].attrs = { 'long_name': 'Sea Surface Temperature', 'standard_name': 'sea_surface_temperature', 'comment': ('Normally this would be sea surface temperature data from a co-located CTD. However, data ' + 'from that sensor is unavailable. This value has been filled with NaNs to preserve the ' + 'structure of the data set.'), 'units': 'degree_Celsius', 'data_product_identifier': 'TEMPSRF_L1', 'instrument': (ds.attrs['subsite'] + '-SBD11-06-METBKA000'), 'stream': 'metbk_a_dcl_instrument' } ds['met_salsurf'] = ('time', ds['deployment'] * np.nan) ds['met_salsurf'].attrs = { 'long_name': 'Sea Surface Practical Salinity', 'standard_name': 'sea_surface_salinity', 'units': '1e-3', 'comment': ('Normally this would be sea surface salinity data from a co-located CTD. However, data from ' + 'that sensor is unavailable. This value has been filled with NaNs to preserve the structure ' + 'of the data set.'), 'data_product_identifier': 'SALSURF_L2', 'instrument': (ds.attrs['subsite'] + '-SBD11-06-METBKA000'), 'stream': 'metbk_a_dcl_instrument' } ds['met_wind10m'] = ('time', ds['deployment'] * np.nan) ds['met_wind10m'].attrs = { 'long_name': 'Normalized Wind Speed at 10 m', 'standard_name': 'wind_speed', 'units': 'm s-1', 'comment': ('Normally this would be the modelled wind speed at a reference height of 10 m from a ' + 'co-located wind sensor. However, data from that sensor is unavailable. This value has been ' + 'filled with NaNs to preserve the structure of the data set.'), 'data_product_identifier': 'WIND10M_L2', 'instrument': (ds.attrs['subsite'] + '-SBD11-06-METBKA000'), 'stream': 'metbk_hourly' } # drop the two QC tests applied to the L0 values (not supposed to happen) if re.match(r'.*_air.*', ds.attrs['stream']): # air stream ds = ds.drop( ['measured_air_co2_qc_executed', 'measured_air_co2_qc_results']) else: # water stream ds = ds.drop([ 'measured_water_co2_qc_executed', 'measured_water_co2_qc_results' ]) # convert the internal timestamp values from a datetime64[ns] object to a floating point number, time in seconds ds['internal_timestamp'] = ('time', dt64_epoch(ds.internal_timestamp)) ds['internal_timestamp'].attrs = dict({ 'long_name': 'Internal pCO2-Pro Clock Time', 'standard_name': 'time', 'units': 'seconds since 1970-01-01 00:00:00 0:00', 'calendar': 'gregorian', 'comment': ('Comparing the instrument internal clock versus the GPS referenced sampling time will allow for ' + 'calculations of the instrument clock offset and drift.') }) # rename variables to get a cleaner set variables and attributes rename = { 'met_salsurf': 'sea_surface_salinity', 'met_wind10m': 'normalized_10m_wind', 'pco2_co2flux': 'sea_air_co2_flux', 'pco2_co2flux_qc_executed': 'sea_air_co2_flux_qc_executed', 'pco2_co2flux_qc_results': 'sea_air_co2_flux_qc_results' } ds = ds.rename(rename) for key, value in rename.items(): # bulk attribute update... ds[value].attrs['ooinet_variable_name'] = key ds['sea_air_co2_flux'].attrs['ancillary_variables'] = ( 'partial_pressure_co2_atm partial_pressure_co2_ssw ' + 'sea_surface_temperature sea_surface_salinity ' + 'normalized_10m_wind sea_air_co2_flux_qc_executed ' + 'sea_air_co2_flux_qc_results') # reset incorrectly formatted temperature units temp_vars = [ 'sea_surface_temperature', 'avg_irga_temperature', 'humidity_temperature', 'irga_detector_temperature', 'irga_source_temperature' ] for var in temp_vars: ds[var].attrs['units'] = 'degree_Celsius' # reset incorrectly set attributes for salinity and wind speed ds['sea_surface_salinity'].attrs['standard_name'] = 'sea_surface_salinity' ds['sea_surface_salinity'].attrs[ 'long_name'] = 'Sea Surface Practical Salinity' ds['sea_surface_salinity'].attrs['units'] = '1e-3' ds['normalized_10m_wind'].attrs['standard_name'] = 'wind_speed' ds['normalized_10m_wind'].attrs[ 'long_name'] = 'Normalized Wind Speed at 10 m' if burst: # re-sample the data to a defined time interval using a median average burst = ds # make a copy of the original dataset burst['time'] = burst['time'].dt.round( 'H') # reset the time values to the nearest hour burst = burst.resample( time='1H', keep_attrs=True, skipna=True).median() # median average the hourly bursts burst = burst.where(~np.isnan(burst.deployment), drop=True) # reset the attributes...which keep_attrs should do... burst.attrs = ds.attrs for v in burst.variables: burst[v].attrs = ds[v].attrs # save the newly average data ds = burst return ds
def ctdbp_datalogger(ds, burst=False): """ Takes ctdbp data recorded by the data loggers used in the CGSN/EA moorings and cleans up the data set to make it more user-friendly. Primary task is renaming the alphabet soup parameter names and dropping some parameters that are of no use/value. :param ds: initial ctdbp data set downloaded from OOI via the M2M system :param burst: resample the data to the defined time interval :return ds: cleaned up data set """ # drop some of the variables: # dcl_controller_timestamp == time, redundant so can remove # date_time_string = internal_timestamp, redundant so can remove ds = ds.reset_coords() ds = ds.drop(['dcl_controller_timestamp', 'date_time_string']) # convert the time values from a datetime64[ns] object to a floating point number with the time in seconds ds['internal_timestamp'] = ('time', dt64_epoch(ds.internal_timestamp)) ds['internal_timestamp'].attrs = dict({ 'long_name': 'Internal CTD Clock Time', 'standard_name': 'time', 'units': 'seconds since 1970-01-01 00:00:00 0:00', 'calendar': 'gregorian', 'comment': ('Comparing the instrument internal clock versus the GPS referenced sampling time will allow for ' + 'calculations of the instrument clock offset and drift. Useful when working with the ' + 'recovered instrument data where no external GPS referenced clock is available.' ) }) # rename some of the variables for better clarity rename = { 'temp': 'temperature', 'temp_qc_executed': 'temperature_qc_executed', 'temp_qc_results': 'temperature_qc_results' } ds = ds.rename(rename) for key, value in rename.items(): ds[value].attrs['ooinet_variable_name'] = key # correct incorrect units ds['temperature'].attrs['units'] = 'degree_Celsius' # ancillary_variables attribute set incorrectly (should be a space separated list) for certain variables ds['temperature'].attrs[ 'ancillary_variables'] = 'temperature_qc_executed temperature_qc_results' ds['conductivity'].attrs[ 'ancillary_variables'] = 'conductivity_qc_executed conductivity_qc_results' ds['pressure'].attrs[ 'ancillary_variables'] = 'pressure_qc_executed pressure_qc_results' ds['practical_salinity'].attrs['ancillary_variables'] = ( 'conductivity temperature pressure ' + 'practical_salinity_qc_executed ' + 'practical_salinity_qc_results') ds['density'].attrs['ancillary_variables'] = ( 'conductivity temperature pressure lat lon ' + 'density_qc_executed density_qc_results') if burst: # re-sample the data to a defined time interval using a median average # create the burst averaging burst = ds burst['time'] = burst['time'] - np.timedelta64( 450, 's') # center time windows for 15 minute bursts burst = burst.resample(time='15Min', keep_attrs=True, skipna=True).median() burst = burst.where(~np.isnan(burst.deployment), drop=True) # reset the attributes...which keep_attrs should do... burst.attrs = ds.attrs for v in burst.variables: burst[v].attrs = ds[v].attrs # save the newly average data ds = burst return ds
def pco2w_datalogger(ds): """ Takes PCO2W data recorded by the data loggers used in the CGSN/EA moorings and cleans up the data set to make it more user-friendly. Primary task is renaming parameters and dropping some that are of limited use. Additionally, re-organize some of the variables to permit better assessments of the data. :param ds: initial PCO2W data set recorded by the data logger system and downloaded from OOI via the M2M system :return: cleaned up and reorganized data set """ # drop some of the variables: # passed_checksum == not used # record_type == not used # record_time == not used # dcl_controller_timestamp == time, redundant so can remove # absorbance_ratio_*_qc_results == incorrectly set tests, ignoring # absorbance_ratio_*_qc_executed == incorrectly set tests, ignoring ds = ds.reset_coords() ds = ds.drop([ 'passed_checksum', 'record_type', 'record_time', 'dcl_controller_timestamp', 'absorbance_ratio_434_qc_results', 'absorbance_ratio_434_qc_executed', 'absorbance_ratio_620_qc_results', 'absorbance_ratio_620_qc_executed' ]) # convert the time values from a datetime64[ns] object to a floating point number with the time in seconds ds['internal_timestamp'] = ('time', dt64_epoch(ds.internal_timestamp)) ds['internal_timestamp'].attrs = dict({ 'long_name': 'Internal SAMI-pCO2 Clock Time', 'standard_name': 'time', 'units': 'seconds since 1970-01-01 00:00:00 0:00', 'calendar': 'gregorian', 'comment': ('Comparing the instrument internal clock versus the GPS referenced sampling time will allow for ' 'calculations of the instrument clock offset and drift. Useful when working with the ' 'recovered instrument data where no external GPS referenced clock is available.' ) }) # check for missing blank data, stripped from the record and treated as a co-located sensor. if 'absorbance_blank_434' not in ds.variables: ds['absorbance_blank_434'] = ('time', ds['deployment'] * 0 - 9999999) ds['absorbance_blank_620'] = ('time', ds['deployment'] * 0 - 9999999) # rename some of the variables for better clarity rename = { 'voltage_battery': 'raw_battery_voltage', 'thermistor_raw': 'raw_thermistor', 'pco2w_thermistor_temperature': 'thermistor_temperature', 'pco2w_thermistor_temperature_qc_executed': 'thermistor_temperature_qc_executed', 'pco2w_thermistor_temperature_qc_results': 'thermistor_temperature_qc_results', } ds = ds.rename(rename) # now we need to reset the light array to named variables that will be more meaningful and useful in # the final data files light = ds.light_measurements.astype('int32') dark_reference = light[:, [0, 8]].values # dark reference dark_signal = light[:, [1, 9]].values # dark signal reference_434 = light[:, [2, 10]].values # reference signal, 434 nm signal_434 = light[:, [3, 11]].values # signal intensity, 434 nm reference_620 = light[:, [4, 12]].values # reference signal, 620 nm signal_620 = light[:, [5, 13]].values # signal intensity, 620 nm # create a data set with the duplicate measurements for each variable data = xr.Dataset( { 'dark_reference': (['time', 'duplicates'], dark_reference), 'dark_signal': (['time', 'duplicates'], dark_signal), 'reference_434': (['time', 'duplicates'], reference_434), 'signal_434': (['time', 'duplicates'], signal_434), 'reference_620': (['time', 'duplicates'], reference_620), 'signal_620': (['time', 'duplicates'], signal_620) }, coords={ 'time': ds['time'], 'duplicates': np.arange(0, 2).astype('int32') }) ds = ds.drop(['spectrum', 'light_measurements']) # merge the data sets back together ds = ds.merge(data) # calculate the battery voltage ds['battery_voltage'] = ds['raw_battery_voltage'] * 15. / 4096. # reset some of the data types data_types = [ 'deployment', 'raw_thermistor', 'unique_id', 'raw_battery_voltage', 'absorbance_blank_434', 'absorbance_blank_620', 'absorbance_ratio_434', 'absorbance_ratio_620' ] for v in data_types: ds[v] = ds[v].astype('int32') data_types = ['thermistor_temperature', 'pco2_seawater'] for v in data_types: ds[v] = ds[v].astype('float32') # test the data quality ds['pco2_seawater_quality_flag'] = quality_checks(ds) # reset some attributes for key, value in ATTRS.items(): for atk, atv in value.items(): if key in ds.variables: ds[key].attrs[atk] = atv # add the original variable name as an attribute, if renamed for key, value in rename.items(): ds[value].attrs['ooinet_variable_name'] = key return ds