extcsv.add_data('DAILY', '2014-04-11,0,0,274,,,,21') extcsv.add_data('DAILY', '2014-04-12,0,2,271,,,,18') extcsv.add_data('DAILY', '2014-04-13,0,2,274,,,,18') extcsv.add_data('DAILY', '2014-04-14,0,0,283,,,,23') extcsv.add_data('DAILY', '2014-04-15,0,0,285,,,,23') extcsv.add_data('DAILY', '2014-04-16,0,0,284,,,,23') extcsv.add_data('DAILY', '2014-04-17,0,0,280,,,,22') extcsv.add_data('DAILY', '2014-04-18,0,2,268,,,,18') extcsv.add_data('DAILY', '2014-04-19,0,2,271,,,,18') extcsv.add_data('DAILY', '2014-04-20,0,2,264,,,,18') extcsv.add_data('DAILY', '2014-04-21,0,0,278,,,,23') extcsv.add_data('DAILY', '2014-04-22,0,0,276,,,,21') extcsv.add_data('DAILY', '2014-04-23,0,0,280,,,,23') extcsv.add_data('DAILY', '2014-04-24,0,0,269,,,,22') extcsv.add_data('DAILY', '2014-04-25,0,0,275,,,,21') extcsv.add_data('DAILY', '2014-04-26,0,2,278,,,,18') extcsv.add_data('DAILY', '2014-04-28,0,0,296,,,,21') extcsv.add_data('DAILY', '2014-04-29,0,0,291,,,,23') extcsv.add_data('DAILY', '2014-04-30,0,0,294,,,,21', table_comment=' 1992 Coefficients in use') extcsv.add_data('TIMESTAMP', '+00:00:00,2014-04-30', field='UTCOffset,Date,Time', index=2) extcsv.add_data('MONTHLY', '2014-04-01,279,8.3,23', field='Date,ColumnO3,StdDevO3,Npts') # write out file to disk woudc_extcsv.dump(extcsv, 'totalozone-extcsv-example1.csv')
def test_dump_file(self): """Test file dumping""" with self.assertRaises(TypeError): dump()
ZenAngle,MuValue,AzimAngle,Flag,TempC', index=2) extcsv.add_data('GLOBAL', '290.0,0.000E+00', field='Wavelength,S-Irradiance,Time', index=2) extcsv.add_data('GLOBAL', '290.5,9.000E-07', field='Wavelength,S-Irradiance,Time', index=2) extcsv.add_data('GLOBAL', '291.0,0.000E+00', field='Wavelength,S-Irradiance,Time', index=2) extcsv.add_data('GLOBAL', '291.5,4.100E-06', field='Wavelength,S-Irradiance,Time', index=2) extcsv.add_data('GLOBAL_DAILY_SUMMARY', '1.794E+02,1.387E+03', field='IntACGIH,IntCIE') extcsv.add_field('GLOBAL_DAILY_SUMMARY', 'IntACGIH,IntCIE') extcsv.add_comment('This is a spectral file') extcsv.add_comment('This is a file comment.') # write out file to disk woudc_extcsv.dump(extcsv, 'general-extcsv-example2.csv')
def test_writer_reader(self): """ Produce woudc_extcsv.Writer object, use woudc_extcsv.Reader to check values """ # produce extcsv object extcsv = Writer(template=True) # add data here extcsv.add_comment('This file was generated by\ WODC_TO_CSX v1.0 using WODC 80-column formatted data.') extcsv.add_comment('\'na\' is used where Instrument\ Model or Number are not available.') extcsv.add_data('CONTENT', 'WOUDC,TotalOzone,1.0,1') extcsv.add_data('DATA_GENERATION', '2014-08-28,NOAA-CMDL,0.0') extcsv.add_data('PLATFORM', 'STN,031,MAUNA LOA,USA') extcsv.add_data('INSTRUMENT', 'Dobson,Beck,076') extcsv.add_data('LOCATION', '19.533,-155.574,3405') extcsv.add_data('TIMESTAMP', '+00:00:00,2014-04-01') extcsv.add_data('DAILY', '2014-04-01,0,2,283,,,,18', field='Date,WLCode,ObsCode,ColumnO3,StdDevO3,\ UTC_Begin,UTC_End,UTC_Mean,nObs,mMu,ColumnSO2') extcsv.add_data('DAILY', '2014-04-08,0,0,288,,,,23') extcsv.add_data('DAILY', '2014-04-09,0,0,279,,,,23') extcsv.add_data('DAILY', '2014-04-10,0,0,273,,,,24') extcsv.add_data('DAILY', '2014-04-11,0,0,274,,,,21') extcsv.add_data('DAILY', '2014-04-12,0,2,271,,,,18') extcsv.add_data('DAILY', '2014-04-13,0,2,274,,,,18') extcsv.add_data('DAILY', '2014-04-14,0,0,283,,,,23') extcsv.add_data('DAILY', '2014-04-15,0,0,285,,,,23') extcsv.add_data('DAILY', '2014-04-16,0,0,284,,,,23') extcsv.add_data('DAILY', '2014-04-17,0,0,280,,,,22') extcsv.add_data('DAILY', '2014-04-18,0,2,268,,,,18') extcsv.add_data('DAILY', '2014-04-19,0,2,271,,,,18') extcsv.add_data('DAILY', '2014-04-20,0,2,264,,,,18') extcsv.add_data('DAILY', '2014-04-21,0,0,278,,,,23') extcsv.add_data('DAILY', '2014-04-22,0,0,276,,,,21') extcsv.add_data('DAILY', '2014-04-23,0,0,280,,,,23') extcsv.add_data('DAILY', '2014-04-24,0,0,269,,,,22') extcsv.add_data('DAILY', '2014-04-25,0,0,275,,,,21') extcsv.add_data('DAILY', '2014-04-26,0,2,278,,,,18') extcsv.add_data('DAILY', '2014-04-28,0,0,296,,,,21') extcsv.add_data('DAILY', '2014-04-29,0,0,291,,,,23') extcsv.add_data('DAILY', '2014-04-30,0,0,294,,,,21', table_comment=' 1992 Coefficients in use') extcsv.add_data('TIMESTAMP', '+00:00:00,2014-04-30', field='UTCOffset,Date,Time', index=2) extcsv.add_data('MONTHLY', '2014-04-01,279,8.3,23', field='Date,ColumnO3,StdDevO3,Npts') extcsv_s = dumps(extcsv) # load my extcsv into Reader my_extcsv_to = loads(extcsv_s) # check tables self.assertTrue('DAILY' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('PLATFORM' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('LOCATION' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('TIMESTAMP' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('DATA_GENERATION' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('TIMESTAMP2' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('MONTHLY' in my_extcsv_to.sections, 'check totalozone in my extcsv') # check fields self.assertTrue('Level' in my_extcsv_to.sections['CONTENT'], 'check totalozone field in my extcsv') self.assertTrue('UTCOffset' in my_extcsv_to.sections['TIMESTAMP'], 'check totalozone field in my extcsv') self.assertTrue( 'ScientificAuthority' in my_extcsv_to.sections['DATA_GENERATION'], 'check totalozone field in my extcsv') self.assertTrue('Time' in my_extcsv_to.sections['TIMESTAMP2'], 'check totalozone field in my extcsv') self.assertTrue('ColumnO3' in my_extcsv_to.sections['MONTHLY'], 'check totalozone field in my extcsv') # check values self.assertEqual('19.533', my_extcsv_to.sections['LOCATION']['Latitude'], 'check totalozone value in my extcsv') self.assertEqual('NOAA-CMDL', my_extcsv_to.sections['DATA_GENERATION']['Agency'], 'check totalozone value in my extcsv') self.assertEqual('1', my_extcsv_to.sections['CONTENT']['Form'], 'check totalozone value in my extcsv') self.assertEqual('23', my_extcsv_to.sections['MONTHLY']['Npts'], 'check totalozone value in my extcsv') my_to_daily = StringIO(my_extcsv_to.sections['DAILY']['_raw']) my_daily_rows = csv.reader(my_to_daily) my_daily_header = my_daily_rows.next() self.assertTrue('WLCode' in my_daily_header, 'check totalozone daily field in my extcsv') self.assertTrue('nObs' in my_daily_header, 'check totalozone daily field in my extcsv') self.assertTrue('ColumnO3' in my_daily_header, 'check totalozone daily field in my extcsv') self.assertTrue('ColumnSO2' in my_daily_header, 'check totalozone daily field in my extcsv') self.assertEqual(0, my_daily_header.index('Date'), 'check totalozone daily field order in my extcsv') self.assertEqual( len(my_daily_header) - 1, my_daily_header.index('ColumnSO2'), 'check totalozone daily field order in my extcsv') my_daily_row = None # seek for i in range(1, 6): my_daily_row = my_daily_rows.next() self.assertEqual('274', my_daily_row[my_daily_header.index('ColumnO3')], 'check totalozone daily value in my extcsv') self.assertEqual('', my_daily_row[my_daily_header.index('StdDevO3')], 'check totalozone daily value in my extcsv') self.assertEqual('', my_daily_row[my_daily_header.index('UTC_Begin')], 'check totalozone daily value in my extcsv') self.assertEqual('21', my_daily_row[my_daily_header.index('UTC_Mean')], 'check totalozone daily value in my extcsv') for i in range(1, 18): my_daily_row = my_daily_rows.next() self.assertEqual('291', my_daily_row[my_daily_header.index('ColumnO3')], 'check totalozone daily value in my extcsv') self.assertEqual('23', my_daily_row[my_daily_header.index('UTC_Mean')], 'check totalozone daily value in my extcsv') with self.assertRaises(TypeError): extcsv_s = dump(extcsv)
extcsv.add_data('FLIGHT_SUMMARY', flight_summary, field=flight_field) # # OZONE_REFERENCE -> O3Ref ozoneref_field = 'Name, Model, Number, Version, TotalO3, WLCode, ObsType, UTC_Mean' df_names = 'O3Ref_Name', 'O3Ref_Model', 'O3Ref_Number', 'O3Ref_Version', 'TotalO3_Col2A', 'WLCode', 'ObsType', 'UTC_Mean' ozoneref_summary = make_summary(dfm,df_names) extcsv.add_data('OZONE_REFERENCE', ozoneref_summary, field=ozoneref_field) # # # PROFILE data_names = 'Duration, Height, Pressure, Temperature, Humidity, TemperatureSonde, O3PartialPressure, SondeCurrent,O3PartialPressure_Uncertainty' df_names = ['Time', 'Height','Pair', 'T', 'U', 'Tbox', 'O3', 'I', 'dO3'] size = len(df) profile = [0] * size for k in range(size): profile[k] = df[df_names][k:k + 1].values[0] profile[k] = ",".join([str(i) for i in profile[k] if str(i)]) extcsv.add_data('#PROFILE', profile[k], field= data_names) out_name = path + str(df.at[df.first_valid_index(),'Date']) + '_testwoudc.csv' print(out_name) woudc_extcsv.dump(extcsv, out_name)
def test_writer_reader(self): """ Produce woudc_extcsv.Writer object, use woudc_extcsv.Reader to check values """ # produce extcsv object extcsv = Writer(template=True) # add data here extcsv.add_comment('This file was generated by\ WODC_TO_CSX v1.0 using WODC 80-column formatted data.') extcsv.add_comment('\'na\' is used where Instrument\ Model or Number are not available.') extcsv.add_data('CONTENT', 'WOUDC,TotalOzone,1.0,1') extcsv.add_data('DATA_GENERATION', '2014-08-28,NOAA-CMDL,0.0') extcsv.add_data('PLATFORM', 'STN,031,MAUNA LOA,USA') extcsv.add_data('INSTRUMENT', 'Dobson,Beck,076') extcsv.add_data('LOCATION', '19.533,-155.574,3405') extcsv.add_data('TIMESTAMP', '+00:00:00,2014-04-01') extcsv.add_data('DAILY', '2014-04-01,0,2,283,,,,18', field='Date,WLCode,ObsCode,ColumnO3,StdDevO3,\ UTC_Begin,UTC_End,UTC_Mean,nObs,mMu,ColumnSO2') extcsv.add_data('DAILY', '2014-04-08,0,0,288,,,,23') extcsv.add_data('DAILY', '2014-04-09,0,0,279,,,,23') extcsv.add_data('DAILY', '2014-04-10,0,0,273,,,,24') extcsv.add_data('DAILY', '2014-04-11,0,0,274,,,,21') extcsv.add_data('DAILY', '2014-04-12,0,2,271,,,,18') extcsv.add_data('DAILY', '2014-04-13,0,2,274,,,,18') extcsv.add_data('DAILY', '2014-04-14,0,0,283,,,,23') extcsv.add_data('DAILY', '2014-04-15,0,0,285,,,,23') extcsv.add_data('DAILY', '2014-04-16,0,0,284,,,,23') extcsv.add_data('DAILY', '2014-04-17,0,0,280,,,,22') extcsv.add_data('DAILY', '2014-04-18,0,2,268,,,,18') extcsv.add_data('DAILY', '2014-04-19,0,2,271,,,,18') extcsv.add_data('DAILY', '2014-04-20,0,2,264,,,,18') extcsv.add_data('DAILY', '2014-04-21,0,0,278,,,,23') extcsv.add_data('DAILY', '2014-04-22,0,0,276,,,,21') extcsv.add_data('DAILY', '2014-04-23,0,0,280,,,,23') extcsv.add_data('DAILY', '2014-04-24,0,0,269,,,,22') extcsv.add_data('DAILY', '2014-04-25,0,0,275,,,,21') extcsv.add_data('DAILY', '2014-04-26,0,2,278,,,,18') extcsv.add_data('DAILY', '2014-04-28,0,0,296,,,,21') extcsv.add_data('DAILY', '2014-04-29,0,0,291,,,,23') extcsv.add_data('DAILY', '2014-04-30,0,0,294,,,,21', table_comment=' 1992 Coefficients in use') extcsv.add_data('TIMESTAMP', '+00:00:00,2014-04-30', field='UTCOffset,Date,Time', index=2) extcsv.add_data('MONTHLY', '2014-04-01,279,8.3,23', field='Date,ColumnO3,StdDevO3,Npts') extcsv_s = dumps(extcsv) # load my extcsv into Reader my_extcsv_to = loads(extcsv_s) # check tables self.assertTrue('DAILY' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('PLATFORM' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('LOCATION' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('TIMESTAMP' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('DATA_GENERATION' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('TIMESTAMP2' in my_extcsv_to.sections, 'check totalozone table in my extcsv') self.assertTrue('MONTHLY' in my_extcsv_to.sections, 'check totalozone in my extcsv') # check fields self.assertTrue('Level' in my_extcsv_to.sections['CONTENT'], 'check totalozone field in my extcsv') self.assertTrue('UTCOffset' in my_extcsv_to.sections['TIMESTAMP'], 'check totalozone field in my extcsv') self.assertTrue('ScientificAuthority' in my_extcsv_to.sections['DATA_GENERATION'], 'check totalozone field in my extcsv') self.assertTrue('Time' in my_extcsv_to.sections['TIMESTAMP2'], 'check totalozone field in my extcsv') self.assertTrue('ColumnO3' in my_extcsv_to.sections['MONTHLY'], 'check totalozone field in my extcsv') # check values self.assertEqual('19.533', my_extcsv_to.sections['LOCATION']['Latitude'], 'check totalozone value in my extcsv') self.assertEqual('NOAA-CMDL', my_extcsv_to.sections['DATA_GENERATION']['Agency'], 'check totalozone value in my extcsv') self.assertEqual('1', my_extcsv_to.sections['CONTENT']['Form'], 'check totalozone value in my extcsv') self.assertEqual('23', my_extcsv_to.sections['MONTHLY']['Npts'], 'check totalozone value in my extcsv') my_to_daily = StringIO(my_extcsv_to.sections['DAILY']['_raw']) my_daily_rows = csv.reader(my_to_daily) my_daily_header = my_daily_rows.next() self.assertTrue('WLCode' in my_daily_header, 'check totalozone daily field in my extcsv') self.assertTrue('nObs' in my_daily_header, 'check totalozone daily field in my extcsv') self.assertTrue('ColumnO3' in my_daily_header, 'check totalozone daily field in my extcsv') self.assertTrue('ColumnSO2' in my_daily_header, 'check totalozone daily field in my extcsv') self.assertEqual(0, my_daily_header.index('Date'), 'check totalozone daily field order in my extcsv') self.assertEqual(len(my_daily_header) - 1, my_daily_header.index('ColumnSO2'), 'check totalozone daily field order in my extcsv') my_daily_row = None # seek for i in range(1, 6): my_daily_row = my_daily_rows.next() self.assertEqual('274', my_daily_row[my_daily_header.index('ColumnO3')], 'check totalozone daily value in my extcsv') self.assertEqual('', my_daily_row[my_daily_header.index('StdDevO3')], 'check totalozone daily value in my extcsv') self.assertEqual('', my_daily_row[my_daily_header.index('UTC_Begin')], 'check totalozone daily value in my extcsv') self.assertEqual('21', my_daily_row[my_daily_header.index('UTC_Mean')], 'check totalozone daily value in my extcsv') for i in range(1, 18): my_daily_row = my_daily_rows.next() self.assertEqual('291', my_daily_row[my_daily_header.index('ColumnO3')], 'check totalozone daily value in my extcsv') self.assertEqual('23', my_daily_row[my_daily_header.index('UTC_Mean')], 'check totalozone daily value in my extcsv') with self.assertRaises(TypeError): extcsv_s = dump(extcsv)
extcsv.remove_data('CONTENT', 'Category', data=10, index=3, all_occurences=True) # clear file # extcsv.clear_file() extcsv.clear_table('CONTENT', index=3) extcsv.add_data('CONTENT', 'new_value', index=3, field='Category') extcsv.clear_field('CONTENT', index=3, field='Category') # take a look at a table extcsv.inspect_table('CONTENT', index=3) # Add file level comments extcsv.add_comment('Comment1') extcsv.add_comment('Comment2') extcsv.add_comment('Comment3') ''' Write to file. By default, the extcsv object will be validated for common/metadata tables and fields. This file is missing some metadata tables and fields, thus file will not serialize. Violations will be printed to standard out and logged. ''' woudc_extcsv.dump(extcsv, 'general-extcsv-example1.csv')