def do2_dofst_volt(voltage_counts, Voffset, Soc, A, B, C, E, P, T, SP, lat, lon): """do2_dofst_volt Takes voltage counts measured from a DOFST-A (SBE 43) Oxygen sensor attached to a CTDPF-A (SBE 16+ V2) CTD, and converts the counts to a voltage and then voltage to dissolved oxygen in units of micromoles/kg for the OOI level 2 data product DOCONCF L2 (fast response oxygen) in combination with salinity, temperature, and pressure from the CTD. A Wrapper function for "dofst_calc". Usage: DO = do2_dofst_volt(volt_counts,Voffset,Soc,A,B,C,E,P,T,SP,lat,lon) where DO = corrected dissolved oxygen [micro-mole/kg], DOCONCF_L2 volt_counts = Oxygen sensor voltage counts [counts], DOCONCF_L0 Voffset = Voltage offset [V]. Soc = Oxygen signal slope [units are inherently inverse volts] A = Residual temperature correction factor A B = Residual temperature correction factor B C = Residual temperature correction factor C E = Pressure correction factor P = PRESWAT water pressure [dbar]. (see 1341-00020_Data_Product_Spec_PRESWAT) T = TEMPWAT water temperature [deg C]. (see 1341-00010_Data_Product_Spec_TEMPWAT) SP = PRACSAL practical salinity [unitless]. (see 1341-00040_Data_Product_Spec_PRACSAL) lat, lon = latitude and longitude of the instrument [degrees]. Example: v_counts = 16384 P = 201.2 T = 30.3 SP = 31.2 lat,lon = 39.0, -70.5 A = -3.1867e-3, B = 1.7749e-4, C = -3.5718e-6 E = 0.036, Voffset = -0.5186, Soc = 0.4396 DO = do2_dofst_volt(v_counts,Voffset,Soc,A,B,C,E,P,T,SP,lat,lon) print DO > 61.89990653 See Also: dofst_calc Implemented by: 2013-08-20: Stuart Pearce. Initial Code. 2015-08-05: Russell Desiderio. Added fillvalue conversion to Nan. """ # replace fill values with nan voltage_counts = replace_fill_with_nan(None, voltage_counts) # convert voltage counts to voltss volts = voltage_counts / 13107. do, do_int = dofst_calc(volts, Voffset, Soc, A, B, C, E, P, T, SP, lat, lon) return do
def do2_dofst_frequency(frequency, Foffset, Soc, A, B, C, E, P, T, SP, lat, lon): """do2_dofst_frequency Takes a frequency measured from a DOFST-K (SBE 43F) Oxygen sensor connected to a CTDPF-CKL (SBE 52-MP) profiling CTD, and converts the frequency to dissolved oxygen in units of micromoles/kg for the OOI level 2 data product DOCONCF L2 (fast response oxygen) in combination with salinity, temperature, and pressure from the CTD. A Wrapper function for "dofst_calc". Usage: DO = do2_dofst_frequency(frequency,Foffset,Soc,A,B,C,E,P,T,SP,lat,lon) where DO = corrected dissolved oxygen [micro-mole/kg], DOCONCF_L2. frequency = Oxygen sensor frequency [Hz], DOCONCF_L0. Foffset = Frequency offset [Hz]. Soc = Oxygen signal slope [units are inherently inverse Hz = seconds] A = Residual temperature correction factor A B = Residual temperature correction factor B C = Residual temperature correction factor C E = Pressure correction factor P = PRESWAT water pressure [dbar]. (see 1341-00020_Data_Product_Spec_PRESWAT) T = TEMPWAT water temperature [deg C]. (see 1341-00010_Data_Product_Spec_TEMPWAT) SP = PRACSAL practical salinity [unitless]. (see 1341-00040_Data_Product_Spec_PRACSAL) lat, lon = latitude and longitude of the instrument [degrees]. Example: f = 4354 P = 60.5200 T = 15.5257 SP = 34.1145 lat,lon = 45.0, -125.0 A = -4.1168e-3, B = 2.4818e-4, C = -3.8820e-6 E = 0.036, Foffset = -839.55, Soc = 2.9968e-4 DO = do2_dofst_frequency(f,Foffset,Soc,A,B,C,E,P,T,SP,lat,lon) print DO > 256.97434863158 See Also: dofst_calc Implemented by: 2013-08-20: Stuart Pearce. Initial Code. 2015-08-05: Russell Desiderio. Added fillvalue conversion to Nan. """ # replace fill values with nan frequency = replace_fill_with_nan(None, frequency) do, do_int = dofst_calc(frequency, Foffset, Soc, A, B, C, E, P, T, SP, lat, lon) return do
def do2_dofst_frequency(frequency, Foffset, Soc, A, B, C, E, P, T, SP, lat, lon): """do2_dofst_frequency Takes a frequency measured from a DOFST-K (SBE 43F) Oxygen sensor connected to a CTDPF-CKL (SBE 52-MP) profiling CTD, and converts the frequency to dissolved oxygen in units of micromoles/kg for the OOI level 2 data product DOCONCF L2 (fast response oxygen) in combination with salinity, temperature, and pressure from the CTD. A Wrapper function for "dofst_calc". Usage: DO = do2_dofst_frequency(frequency,Foffset,Soc,A,B,C,E,P,T,SP,lat,lon) where DO = corrected dissolved oxygen [micro-mole/kg], DOCONCF_L2. frequency = Oxygen sensor frequency [Hz], DOCONCF_L0. Foffset = Frequency offset [Hz]. Soc = Oxygen signal slope [units are inherently inverse Hz = seconds] A = Residual temperature correction factor A B = Residual temperature correction factor B C = Residual temperature correction factor C E = Pressure correction factor P = PRESWAT water pressure [dbar]. (see 1341-00020_Data_Product_Spec_PRESWAT) T = TEMPWAT water temperature [deg C]. (see 1341-00010_Data_Product_Spec_TEMPWAT) SP = PRACSAL practical salinity [unitless]. (see 1341-00040_Data_Product_Spec_PRACSAL) lat, lon = latitude and longitude of the instrument [degrees]. Example: f = 4354 P = 60.5200 T = 15.5257 SP = 34.1145 lat,lon = 45.0, -125.0 A = -4.1168e-3, B = 2.4818e-4, C = -3.8820e-6 E = 0.036, Foffset = -839.55, Soc = 2.9968e-4 DO = do2_dofst_frequency(f,Foffset,Soc,A,B,C,E,P,T,SP,lat,lon) print DO > 256.97434863158 See Also: dofst_calc Implemented by: 2013-08-20: Stuart Pearce. Initial Code. 2015-08-05: Russell Desiderio. Added fillvalue conversion to Nan. """ # replace fill values with nan frequency = replace_fill_with_nan(None, frequency) do, do_int = dofst_calc(frequency, Foffset, Soc, A, B, C, E, P, T, SP, lat, lon) return do
def o2_counts_to_uM(o2_counts): """ Description: Computes the DOCONCS_L1 data product from a DOSTA Aanderaa Optode connected to a SBE 16+ V2 CTD via RS-232. Usage: DO = o2_counts_to_uM(o2_counts) where DO = DOCONCS_L1, dissolved oxygen concentration uncorrected for salinity and pressure effects [micro-mole/L] o2_counts = DOCONCS-CNT_L0, counts from the CTD Implemented by: 2013-04-26: Stuart Pearce. Initial Code. 2015-04-10: Russell Desiderio. Added documentation and fill value handling. Notes: The DOCONCS_L1 data product has units of micromole/liter; SAF incorrectly lists the units for this L1 product as micromole/kg. References: OOI (2012). Data Product Specification for Oxygen Concentration from "Stable" Instruments. Document Control Number 1341-00520. https://alfresco.oceanobservatories.org/ (See: Company Home >> OOI >> Controlled >> 1000 System Level >> 1341-00520_Data_Product_SPEC_DOCONCS_OOI.pdf) """ # replace fill values with nan o2_counts = replace_fill_with_nan(None, o2_counts) DO = (o2_counts / 10000.0) - 10.0 return DO
def o2_counts_to_uM(o2_counts): """ Description: Computes the DOCONCS_L1 data product from a DOSTA Aanderaa Optode connected to a SBE 16+ V2 CTD via RS-232. Usage: DO = o2_counts_to_uM(o2_counts) where DO = DOCONCS_L1, dissolved oxygen concentration uncorrected for salinity and pressure effects [micro-mole/L] o2_counts = DOCONCS-CNT_L0, counts from the CTD Implemented by: 2013-04-26: Stuart Pearce. Initial Code. 2015-04-10: Russell Desiderio. Added documentation and fill value handling. Notes: The DOCONCS_L1 data product has units of micromole/liter; SAF incorrectly lists the units for this L1 product as micromole/kg. References: OOI (2012). Data Product Specification for Oxygen Concentration from "Stable" Instruments. Document Control Number 1341-00520. https://alfresco.oceanobservatories.org/ (See: Company Home >> OOI >> Controlled >> 1000 System Level >> 1341-00520_Data_Product_SPEC_DOCONCS_OOI.pdf) """ # replace fill values with nan o2_counts = replace_fill_with_nan(None, o2_counts) DO = (o2_counts / 10000.0) - 10.0 return DO
def do2_dofst_volt(voltage_counts, Voffset, Soc, A, B, C, E, P, T, SP, lat, lon): """do2_dofst_volt Takes voltage counts measured from a DOFST-A (SBE 43) Oxygen sensor attached to a CTDPF-A (SBE 16+ V2) CTD, and converts the counts to a voltage and then voltage to dissolved oxygen in units of micromoles/kg for the OOI level 2 data product DOCONCF L2 (fast response oxygen) in combination with salinity, temperature, and pressure from the CTD. A Wrapper function for "dofst_calc". Usage: DO = do2_dofst_volt(volt_counts,Voffset,Soc,A,B,C,E,P,T,SP,lat,lon) where DO = corrected dissolved oxygen [micro-mole/kg], DOCONCF_L2 volt_counts = Oxygen sensor voltage counts [counts], DOCONCF_L0 Voffset = Voltage offset [V]. Soc = Oxygen signal slope [units are inherently inverse volts] A = Residual temperature correction factor A B = Residual temperature correction factor B C = Residual temperature correction factor C E = Pressure correction factor P = PRESWAT water pressure [dbar]. (see 1341-00020_Data_Product_Spec_PRESWAT) T = TEMPWAT water temperature [deg C]. (see 1341-00010_Data_Product_Spec_TEMPWAT) SP = PRACSAL practical salinity [unitless]. (see 1341-00040_Data_Product_Spec_PRACSAL) lat, lon = latitude and longitude of the instrument [degrees]. Example: v_counts = 16384 P = 201.2 T = 30.3 SP = 31.2 lat,lon = 39.0, -70.5 A = -3.1867e-3, B = 1.7749e-4, C = -3.5718e-6 E = 0.036, Voffset = -0.5186, Soc = 0.4396 DO = do2_dofst_volt(v_counts,Voffset,Soc,A,B,C,E,P,T,SP,lat,lon) print DO > 61.89990653 See Also: dofst_calc Implemented by: 2013-08-20: Stuart Pearce. Initial Code. 2015-08-05: Russell Desiderio. Added fillvalue conversion to Nan. """ # replace fill values with nan voltage_counts = replace_fill_with_nan(None, voltage_counts) # convert voltage counts to voltss volts = voltage_counts / 13107. do, do_int = dofst_calc(volts, Voffset, Soc, A, B, C, E, P, T, SP, lat, lon) return do
def test_replace_fill_with_nan_with_one_data_argument(self): """ Description: Tests replace_fill_with_nan when called with only one data argument. Coding replace_fill_with_nan for this was initially not straightforward due to the core python\numpy "interface". Cases (instances of the system fill value are always processed): no instrument fill value one instrument fill value of 0 two instrument fill values Implemented by: 2013-06-23: Russell Desiderio. Initial code. """ # constants for convenience in filling in test variables sfill = SYSTEM_FILLVALUE zfill = ZERO_FILLVALUE ifill = INST_FILLVALUE # set up a list of 8 input integer arrays to be tested one at a time v4 = np.array([[1, 2, 3, 4, 5]]) v5 = np.array([[11, ifill, 13, zfill, sfill]]) v = [np.array([500]), np.array([ifill]), np.array([zfill]), np.array([sfill]), v4, v5, np.tile(v4, (3, 1))[np.newaxis, :], np.vstack((v5, v4, v5))[np.newaxis, :] ] ### CASE(1): no instrument fill values # expected values x = [np.array([500.0]), np.array([ifill]).astype('float'), np.array([zfill]).astype('float'), np.array([np.nan]), np.array([[1.0, 2.0, 3.0, 4.0, 5.0]]), np.array([[11, ifill, 13, zfill, np.nan]]).astype('float'), v[6].astype(float), np.copy(v[7].astype(float)) ] x[7][0, 0, 4] = np.nan x[7][0, 2, 4] = np.nan for ii in range(len(v)): c = gfunc.replace_fill_with_nan(None, v[ii]) np.testing.assert_array_almost_equal(c, x[ii], decimal=8) ### CASE (2): one instrument fill value = 0 # reset expected outputs; x[0], x[1], x[3], x[4] and x[6] are unchanged x[2] = np.array([np.nan]) x[5][0, 3] = np.nan # x[5] is now reset x[7][0, 0, 3] = np.nan x[7][0, 2, 3] = np.nan # x[7] is now reset for ii in range(len(v)): c = gfunc.replace_fill_with_nan(zfill, v[ii]) np.testing.assert_array_almost_equal(c, x[ii], decimal=8) ### CASE (3): two instrument fill values # reset expected outputs; only list elements 5 and 7 will change x[1] = np.array([np.nan]) x[5][0, 1] = np.nan # x[5] is now reset x[7][0, 0, 1] = np.nan x[7][0, 2, 1] = np.nan # x[7] is now reset for ii in range(len(v)): c = gfunc.replace_fill_with_nan([ifill, zfill], v[ii]) np.testing.assert_array_almost_equal(c, x[ii], decimal=8)
def test_replace_fill_with_nan_with_vectortime_variables(self): """ Description: Tests replace_fill_with_nan for variables that are time-vectorized with more than one time point. Functionally this means that the lead dimension of these test variables will have a shape greater than 1. Cases (instances of the system fill value are always processed): no instrument fill value one instrument fill value of 0 two instrument fill values Implemented by: 2013-06-11: Russell Desiderio. Initial code. """ # constants for convenience in filling in test variables sfill = SYSTEM_FILLVALUE zfill = ZERO_FILLVALUE ifill = INST_FILLVALUE # set up 6 integer arrays to be tested, v1-v6. # this will also designate their ordering on input and output v1_good_1D = np.array([1, 2, 3, 4, 5]) v2_good_2D = np.tile(v1_good_1D, (3, 1)) v3_good_3D = np.tile(v2_good_2D, (3, 1, 1)) v4_fill_1D = np.array([11, ifill, 13, zfill, sfill]) v5_fill_2D = np.vstack((v4_fill_1D, v1_good_1D, v4_fill_1D)) v6_fill_3D = np.copy(v3_good_3D) v6_fill_3D[0, 0, 0] = zfill v6_fill_3D[1, 1, 1] = sfill v6_fill_3D[2, 2, 2:5] = ifill # last 3 elements in row '2' # and a float array which should pass through unchanged v7_float_array = v3_good_3D.astype(float) v7_float_array[0, 1, 2] = np.nan #SET INPUTS v1, v2, v3 = np.copy(v1_good_1D), np.copy(v2_good_2D), np.copy(v3_good_3D) v4, v5, v6 = np.copy(v4_fill_1D), np.copy(v5_fill_2D), np.copy(v6_fill_3D) v7 = np.copy(v7_float_array) ### CASE 1: no instrument fill values, only system fill values # set expected outputs x1 = v1_good_1D.astype(float) x2 = v2_good_2D.astype(float) x3 = v3_good_3D.astype(float) x4 = np.array([11.0, float(ifill), 13.0, float(zfill), np.nan]) x5 = np.vstack((x4, x1, x4)) x6 = v6_fill_3D.astype(float) x6[1, 1, 1] = np.nan x7 = np.copy(v7_float_array) # (1a): instrument_fillvalue = None (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan( None, v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (1b) instrument_fillvalue = empty list (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan( [], v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (1c) instrument_fillvalue: empty ndarray (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan( np.array([]), v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) ### CASE (2): one instrument fill value = 0 # reset expected outputs; x1-x3 and x7 are unchanged x4[3] = np.nan # x4 is now reset x5 = np.vstack((x4, x1, x4)) x6[0, 0, 0] = np.nan # x6 is now reset # (2a) instrument_fillvalue is a scalar (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan( zfill, v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (2b) instrument_fillvalue is a one-element list (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan( [zfill], v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (2c) instrument_fillvalue is a shape (1,) ndarray (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan( np.array([zfill]), v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) ### CASE (3): two instrument fill values # reset expected outputs; x1-x3 and x7 are unchanged x4[1] = np.nan # x4 is now reset x5 = np.vstack((x4, x1, x4)) x6[2, 2, 2:5] = np.nan # x6 is now reset # (3a) instrument_fillvalues are contained in a list (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan( [ifill, zfill], v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (3b) instrument_fillvalues are contained in an ndarray (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan( np.array([ifill, zfill]), v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8)
def test_replace_fill_with_nan_with_scalartime_variables(self): """ Description: Tests replace_fill_with_nan for variables that are time-vectorized with one time point. Functionally this means that the lead dimension of these test variables will have shape 1. Cases (instances of the system fill value are always processed): no instrument fill value one instrument fill value of 0 two instrument fill values Implemented by: 2013-06-12: Russell Desiderio. Initial code. """ # constants for convenience in filling in test variables sfill = SYSTEM_FILLVALUE zfill = ZERO_FILLVALUE ifill = INST_FILLVALUE # set up 8 integer arrays to be tested, v1-v8. # this will also designate their ordering on input and output v1 = np.array([500]) v2 = np.array([ifill]) v3 = np.array([zfill]) v4 = np.array([sfill]) v5 = np.array([[1, 2, 3, 4, 5]]) v6 = np.array([[11, ifill, 13, zfill, sfill]]) v7 = np.tile(v5, (3, 1))[np.newaxis, :] v8 = np.vstack((v6, v5, v6))[np.newaxis, :] ### CASE 1: no instrument fill values, only system fill values # set expected outputs x1, x2, x3 = v1.astype(float), v2.astype(float), v3.astype(float) x4 = np.array([np.nan]) x5 = v5.astype(float) x6 = np.array([[11.0, float(ifill), 13.0, float(zfill), np.nan]]) x7 = v7.astype(float) x8 = np.copy(v8.astype(float)) x8[0, 0, 4] = np.nan x8[0, 2, 4] = np.nan # (1a) instrument_fillvalue = None (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan( None, v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (1b) instrument_fillvalue: empty list (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan( [], v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (1c) instrument_fillvalue: empty ndarray (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan( np.array([]), v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) ### CASE (2): one instrument fill value = 0 # reset expected outputs; x1, x2, x4, x5 and x7 are unchanged x3 = np.array([np.nan]) x6[0, 3] = np.nan # x6 is now reset x8[0, 0, 3] = np.nan x8[0, 2, 3] = np.nan # x8 is now reset # (2a) instrument_fillvalue is a scalar (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan( zfill, v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (2b) instrument_fillvalue is a one element list (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan( [zfill], v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (2c) instrument_fillvalue is a one element ndarray (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan( np.array([zfill]), v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) ### CASE (3): two instrument fill values # reset expected outputs; x1,x3-x5 and x7 are unchanged x2 = np.array([np.nan]) x6[0, 1] = np.nan # x6 is now reset x8[0, 0, 1] = np.nan x8[0, 2, 1] = np.nan # x8 is now reset # (3a) instrument_fillvalues are contained in a list (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan( [ifill, zfill], v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (3b) instrument_fillvalues are contained in an ndarray (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan( np.array([ifill, zfill]), v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8)
def test_replace_fill_with_nan_with_one_data_argument(self): """ Description: Tests replace_fill_with_nan when called with only one data argument. Coding replace_fill_with_nan for this was initially not straightforward due to the core python\numpy "interface". Cases (instances of the system fill value are always processed): no instrument fill value one instrument fill value of 0 two instrument fill values Implemented by: 2013-06-23: Russell Desiderio. Initial code. """ # constants for convenience in filling in test variables sfill = SYSTEM_FILLVALUE zfill = ZERO_FILLVALUE ifill = INST_FILLVALUE # set up a list of 8 input integer arrays to be tested one at a time v4 = np.array([[1, 2, 3, 4, 5]]) v5 = np.array([[11, ifill, 13, zfill, sfill]]) v = [ np.array([500]), np.array([ifill]), np.array([zfill]), np.array([sfill]), v4, v5, np.tile(v4, (3, 1))[np.newaxis, :], np.vstack((v5, v4, v5))[np.newaxis, :] ] ### CASE(1): no instrument fill values # expected values x = [ np.array([500.0]), np.array([ifill]).astype('float'), np.array([zfill]).astype('float'), np.array([np.nan]), np.array([[1.0, 2.0, 3.0, 4.0, 5.0]]), np.array([[11, ifill, 13, zfill, np.nan]]).astype('float'), v[6].astype(float), np.copy(v[7].astype(float)) ] x[7][0, 0, 4] = np.nan x[7][0, 2, 4] = np.nan for ii in range(len(v)): c = gfunc.replace_fill_with_nan(None, v[ii]) np.testing.assert_array_almost_equal(c, x[ii], decimal=8) ### CASE (2): one instrument fill value = 0 # reset expected outputs; x[0], x[1], x[3], x[4] and x[6] are unchanged x[2] = np.array([np.nan]) x[5][0, 3] = np.nan # x[5] is now reset x[7][0, 0, 3] = np.nan x[7][0, 2, 3] = np.nan # x[7] is now reset for ii in range(len(v)): c = gfunc.replace_fill_with_nan(zfill, v[ii]) np.testing.assert_array_almost_equal(c, x[ii], decimal=8) ### CASE (3): two instrument fill values # reset expected outputs; only list elements 5 and 7 will change x[1] = np.array([np.nan]) x[5][0, 1] = np.nan # x[5] is now reset x[7][0, 0, 1] = np.nan x[7][0, 2, 1] = np.nan # x[7] is now reset for ii in range(len(v)): c = gfunc.replace_fill_with_nan([ifill, zfill], v[ii]) np.testing.assert_array_almost_equal(c, x[ii], decimal=8)
def test_replace_fill_with_nan_with_vectortime_variables(self): """ Description: Tests replace_fill_with_nan for variables that are time-vectorized with more than one time point. Functionally this means that the lead dimension of these test variables will have a shape greater than 1. Cases (instances of the system fill value are always processed): no instrument fill value one instrument fill value of 0 two instrument fill values Implemented by: 2013-06-11: Russell Desiderio. Initial code. """ # constants for convenience in filling in test variables sfill = SYSTEM_FILLVALUE zfill = ZERO_FILLVALUE ifill = INST_FILLVALUE # set up 6 integer arrays to be tested, v1-v6. # this will also designate their ordering on input and output v1_good_1D = np.array([1, 2, 3, 4, 5]) v2_good_2D = np.tile(v1_good_1D, (3, 1)) v3_good_3D = np.tile(v2_good_2D, (3, 1, 1)) v4_fill_1D = np.array([11, ifill, 13, zfill, sfill]) v5_fill_2D = np.vstack((v4_fill_1D, v1_good_1D, v4_fill_1D)) v6_fill_3D = np.copy(v3_good_3D) v6_fill_3D[0, 0, 0] = zfill v6_fill_3D[1, 1, 1] = sfill v6_fill_3D[2, 2, 2:5] = ifill # last 3 elements in row '2' # and a float array which should pass through unchanged v7_float_array = v3_good_3D.astype(float) v7_float_array[0, 1, 2] = np.nan #SET INPUTS v1, v2, v3 = np.copy(v1_good_1D), np.copy(v2_good_2D), np.copy( v3_good_3D) v4, v5, v6 = np.copy(v4_fill_1D), np.copy(v5_fill_2D), np.copy( v6_fill_3D) v7 = np.copy(v7_float_array) ### CASE 1: no instrument fill values, only system fill values # set expected outputs x1 = v1_good_1D.astype(float) x2 = v2_good_2D.astype(float) x3 = v3_good_3D.astype(float) x4 = np.array([11.0, float(ifill), 13.0, float(zfill), np.nan]) x5 = np.vstack((x4, x1, x4)) x6 = v6_fill_3D.astype(float) x6[1, 1, 1] = np.nan x7 = np.copy(v7_float_array) # (1a): instrument_fillvalue = None (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan(None, v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (1b) instrument_fillvalue = empty list (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan([], v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (1c) instrument_fillvalue: empty ndarray (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan(np.array([]), v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) ### CASE (2): one instrument fill value = 0 # reset expected outputs; x1-x3 and x7 are unchanged x4[3] = np.nan # x4 is now reset x5 = np.vstack((x4, x1, x4)) x6[0, 0, 0] = np.nan # x6 is now reset # (2a) instrument_fillvalue is a scalar (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan(zfill, v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (2b) instrument_fillvalue is a one-element list (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan([zfill], v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (2c) instrument_fillvalue is a shape (1,) ndarray (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan(np.array([zfill]), v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) ### CASE (3): two instrument fill values # reset expected outputs; x1-x3 and x7 are unchanged x4[1] = np.nan # x4 is now reset x5 = np.vstack((x4, x1, x4)) x6[2, 2, 2:5] = np.nan # x6 is now reset # (3a) instrument_fillvalues are contained in a list (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan([ifill, zfill], v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) # (3b) instrument_fillvalues are contained in an ndarray (c1, c2, c3, c4, c5, c6, c7) = gfunc.replace_fill_with_nan(np.array([ifill, zfill]), v1, v2, v3, v4, v5, v6, v7) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8)
def test_replace_fill_with_nan_with_scalartime_variables(self): """ Description: Tests replace_fill_with_nan for variables that are time-vectorized with one time point. Functionally this means that the lead dimension of these test variables will have shape 1. Cases (instances of the system fill value are always processed): no instrument fill value one instrument fill value of 0 two instrument fill values Implemented by: 2013-06-12: Russell Desiderio. Initial code. """ # constants for convenience in filling in test variables sfill = SYSTEM_FILLVALUE zfill = ZERO_FILLVALUE ifill = INST_FILLVALUE # set up 8 integer arrays to be tested, v1-v8. # this will also designate their ordering on input and output v1 = np.array([500]) v2 = np.array([ifill]) v3 = np.array([zfill]) v4 = np.array([sfill]) v5 = np.array([[1, 2, 3, 4, 5]]) v6 = np.array([[11, ifill, 13, zfill, sfill]]) v7 = np.tile(v5, (3, 1))[np.newaxis, :] v8 = np.vstack((v6, v5, v6))[np.newaxis, :] ### CASE 1: no instrument fill values, only system fill values # set expected outputs x1, x2, x3 = v1.astype(float), v2.astype(float), v3.astype(float) x4 = np.array([np.nan]) x5 = v5.astype(float) x6 = np.array([[11.0, float(ifill), 13.0, float(zfill), np.nan]]) x7 = v7.astype(float) x8 = np.copy(v8.astype(float)) x8[0, 0, 4] = np.nan x8[0, 2, 4] = np.nan # (1a) instrument_fillvalue = None (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan(None, v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (1b) instrument_fillvalue: empty list (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan([], v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (1c) instrument_fillvalue: empty ndarray (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan(np.array([]), v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) ### CASE (2): one instrument fill value = 0 # reset expected outputs; x1, x2, x4, x5 and x7 are unchanged x3 = np.array([np.nan]) x6[0, 3] = np.nan # x6 is now reset x8[0, 0, 3] = np.nan x8[0, 2, 3] = np.nan # x8 is now reset # (2a) instrument_fillvalue is a scalar (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan(zfill, v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (2b) instrument_fillvalue is a one element list (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan([zfill], v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (2c) instrument_fillvalue is a one element ndarray (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan(np.array([zfill]), v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) ### CASE (3): two instrument fill values # reset expected outputs; x1,x3-x5 and x7 are unchanged x2 = np.array([np.nan]) x6[0, 1] = np.nan # x6 is now reset x8[0, 0, 1] = np.nan x8[0, 2, 1] = np.nan # x8 is now reset # (3a) instrument_fillvalues are contained in a list (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan([ifill, zfill], v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8) # (3b) instrument_fillvalues are contained in an ndarray (c1, c2, c3, c4, c5, c6, c7, c8) = gfunc.replace_fill_with_nan(np.array([ifill, zfill]), v1, v2, v3, v4, v5, v6, v7, v8) np.testing.assert_array_almost_equal(c1, x1, decimal=8) np.testing.assert_array_almost_equal(c2, x2, decimal=8) np.testing.assert_array_almost_equal(c3, x3, decimal=8) np.testing.assert_array_almost_equal(c4, x4, decimal=8) np.testing.assert_array_almost_equal(c5, x5, decimal=8) np.testing.assert_array_almost_equal(c6, x6, decimal=8) np.testing.assert_array_almost_equal(c7, x7, decimal=8) np.testing.assert_array_almost_equal(c8, x8, decimal=8)