extcsv.add_data('DAILY', '2014-04-11,0,0,274,,,,21')
extcsv.add_data('DAILY', '2014-04-12,0,2,271,,,,18')
extcsv.add_data('DAILY', '2014-04-13,0,2,274,,,,18')
extcsv.add_data('DAILY', '2014-04-14,0,0,283,,,,23')
extcsv.add_data('DAILY', '2014-04-15,0,0,285,,,,23')
extcsv.add_data('DAILY', '2014-04-16,0,0,284,,,,23')
extcsv.add_data('DAILY', '2014-04-17,0,0,280,,,,22')
extcsv.add_data('DAILY', '2014-04-18,0,2,268,,,,18')
extcsv.add_data('DAILY', '2014-04-19,0,2,271,,,,18')
extcsv.add_data('DAILY', '2014-04-20,0,2,264,,,,18')
extcsv.add_data('DAILY', '2014-04-21,0,0,278,,,,23')
extcsv.add_data('DAILY', '2014-04-22,0,0,276,,,,21')
extcsv.add_data('DAILY', '2014-04-23,0,0,280,,,,23')
extcsv.add_data('DAILY', '2014-04-24,0,0,269,,,,22')
extcsv.add_data('DAILY', '2014-04-25,0,0,275,,,,21')
extcsv.add_data('DAILY', '2014-04-26,0,2,278,,,,18')
extcsv.add_data('DAILY', '2014-04-28,0,0,296,,,,21')
extcsv.add_data('DAILY', '2014-04-29,0,0,291,,,,23')
extcsv.add_data('DAILY', '2014-04-30,0,0,294,,,,21',
table_comment=' 1992 Coefficients in use')
extcsv.add_data('TIMESTAMP',
'+00:00:00,2014-04-30',
field='UTCOffset,Date,Time',
index=2)
extcsv.add_data('MONTHLY',
'2014-04-01,279,8.3,23',
field='Date,ColumnO3,StdDevO3,Npts')
# write out file to disk
woudc_extcsv.dump(extcsv, 'totalozone-extcsv-example1.csv')
def test_dump_file(self):
"""Test file dumping"""
with self.assertRaises(TypeError):
dump()
ZenAngle,MuValue,AzimAngle,Flag,TempC',
index=2)
extcsv.add_data('GLOBAL',
'290.0,0.000E+00',
field='Wavelength,S-Irradiance,Time',
index=2)
extcsv.add_data('GLOBAL',
'290.5,9.000E-07',
field='Wavelength,S-Irradiance,Time',
index=2)
extcsv.add_data('GLOBAL',
'291.0,0.000E+00',
field='Wavelength,S-Irradiance,Time',
index=2)
extcsv.add_data('GLOBAL',
'291.5,4.100E-06',
field='Wavelength,S-Irradiance,Time',
index=2)
extcsv.add_data('GLOBAL_DAILY_SUMMARY',
'1.794E+02,1.387E+03',
field='IntACGIH,IntCIE')
extcsv.add_field('GLOBAL_DAILY_SUMMARY', 'IntACGIH,IntCIE')
extcsv.add_comment('This is a spectral file')
extcsv.add_comment('This is a file comment.')
# write out file to disk
woudc_extcsv.dump(extcsv, 'general-extcsv-example2.csv')
def test_writer_reader(self):
"""
Produce woudc_extcsv.Writer object,
use woudc_extcsv.Reader to check values
"""
# produce extcsv object
extcsv = Writer(template=True)
# add data here
extcsv.add_comment('This file was generated by\
WODC_TO_CSX v1.0 using WODC 80-column formatted data.')
extcsv.add_comment('\'na\' is used where Instrument\
Model or Number are not available.')
extcsv.add_data('CONTENT', 'WOUDC,TotalOzone,1.0,1')
extcsv.add_data('DATA_GENERATION', '2014-08-28,NOAA-CMDL,0.0')
extcsv.add_data('PLATFORM', 'STN,031,MAUNA LOA,USA')
extcsv.add_data('INSTRUMENT', 'Dobson,Beck,076')
extcsv.add_data('LOCATION', '19.533,-155.574,3405')
extcsv.add_data('TIMESTAMP', '+00:00:00,2014-04-01')
extcsv.add_data('DAILY',
'2014-04-01,0,2,283,,,,18',
field='Date,WLCode,ObsCode,ColumnO3,StdDevO3,\
UTC_Begin,UTC_End,UTC_Mean,nObs,mMu,ColumnSO2')
extcsv.add_data('DAILY', '2014-04-08,0,0,288,,,,23')
extcsv.add_data('DAILY', '2014-04-09,0,0,279,,,,23')
extcsv.add_data('DAILY', '2014-04-10,0,0,273,,,,24')
extcsv.add_data('DAILY', '2014-04-11,0,0,274,,,,21')
extcsv.add_data('DAILY', '2014-04-12,0,2,271,,,,18')
extcsv.add_data('DAILY', '2014-04-13,0,2,274,,,,18')
extcsv.add_data('DAILY', '2014-04-14,0,0,283,,,,23')
extcsv.add_data('DAILY', '2014-04-15,0,0,285,,,,23')
extcsv.add_data('DAILY', '2014-04-16,0,0,284,,,,23')
extcsv.add_data('DAILY', '2014-04-17,0,0,280,,,,22')
extcsv.add_data('DAILY', '2014-04-18,0,2,268,,,,18')
extcsv.add_data('DAILY', '2014-04-19,0,2,271,,,,18')
extcsv.add_data('DAILY', '2014-04-20,0,2,264,,,,18')
extcsv.add_data('DAILY', '2014-04-21,0,0,278,,,,23')
extcsv.add_data('DAILY', '2014-04-22,0,0,276,,,,21')
extcsv.add_data('DAILY', '2014-04-23,0,0,280,,,,23')
extcsv.add_data('DAILY', '2014-04-24,0,0,269,,,,22')
extcsv.add_data('DAILY', '2014-04-25,0,0,275,,,,21')
extcsv.add_data('DAILY', '2014-04-26,0,2,278,,,,18')
extcsv.add_data('DAILY', '2014-04-28,0,0,296,,,,21')
extcsv.add_data('DAILY', '2014-04-29,0,0,291,,,,23')
extcsv.add_data('DAILY',
'2014-04-30,0,0,294,,,,21',
table_comment=' 1992 Coefficients in use')
extcsv.add_data('TIMESTAMP',
'+00:00:00,2014-04-30',
field='UTCOffset,Date,Time',
index=2)
extcsv.add_data('MONTHLY',
'2014-04-01,279,8.3,23',
field='Date,ColumnO3,StdDevO3,Npts')
extcsv_s = dumps(extcsv)
# load my extcsv into Reader
my_extcsv_to = loads(extcsv_s)
# check tables
self.assertTrue('DAILY' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('PLATFORM' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('LOCATION' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('TIMESTAMP' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('DATA_GENERATION' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('TIMESTAMP2' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('MONTHLY' in my_extcsv_to.sections,
'check totalozone in my extcsv')
# check fields
self.assertTrue('Level' in my_extcsv_to.sections['CONTENT'],
'check totalozone field in my extcsv')
self.assertTrue('UTCOffset' in my_extcsv_to.sections['TIMESTAMP'],
'check totalozone field in my extcsv')
self.assertTrue(
'ScientificAuthority' in my_extcsv_to.sections['DATA_GENERATION'],
'check totalozone field in my extcsv')
self.assertTrue('Time' in my_extcsv_to.sections['TIMESTAMP2'],
'check totalozone field in my extcsv')
self.assertTrue('ColumnO3' in my_extcsv_to.sections['MONTHLY'],
'check totalozone field in my extcsv')
# check values
self.assertEqual('19.533',
my_extcsv_to.sections['LOCATION']['Latitude'],
'check totalozone value in my extcsv')
self.assertEqual('NOAA-CMDL',
my_extcsv_to.sections['DATA_GENERATION']['Agency'],
'check totalozone value in my extcsv')
self.assertEqual('1', my_extcsv_to.sections['CONTENT']['Form'],
'check totalozone value in my extcsv')
self.assertEqual('23', my_extcsv_to.sections['MONTHLY']['Npts'],
'check totalozone value in my extcsv')
my_to_daily = StringIO(my_extcsv_to.sections['DAILY']['_raw'])
my_daily_rows = csv.reader(my_to_daily)
my_daily_header = my_daily_rows.next()
self.assertTrue('WLCode' in my_daily_header,
'check totalozone daily field in my extcsv')
self.assertTrue('nObs' in my_daily_header,
'check totalozone daily field in my extcsv')
self.assertTrue('ColumnO3' in my_daily_header,
'check totalozone daily field in my extcsv')
self.assertTrue('ColumnSO2' in my_daily_header,
'check totalozone daily field in my extcsv')
self.assertEqual(0, my_daily_header.index('Date'),
'check totalozone daily field order in my extcsv')
self.assertEqual(
len(my_daily_header) - 1, my_daily_header.index('ColumnSO2'),
'check totalozone daily field order in my extcsv')
my_daily_row = None
# seek
for i in range(1, 6):
my_daily_row = my_daily_rows.next()
self.assertEqual('274',
my_daily_row[my_daily_header.index('ColumnO3')],
'check totalozone daily value in my extcsv')
self.assertEqual('', my_daily_row[my_daily_header.index('StdDevO3')],
'check totalozone daily value in my extcsv')
self.assertEqual('', my_daily_row[my_daily_header.index('UTC_Begin')],
'check totalozone daily value in my extcsv')
self.assertEqual('21', my_daily_row[my_daily_header.index('UTC_Mean')],
'check totalozone daily value in my extcsv')
for i in range(1, 18):
my_daily_row = my_daily_rows.next()
self.assertEqual('291',
my_daily_row[my_daily_header.index('ColumnO3')],
'check totalozone daily value in my extcsv')
self.assertEqual('23', my_daily_row[my_daily_header.index('UTC_Mean')],
'check totalozone daily value in my extcsv')
with self.assertRaises(TypeError):
extcsv_s = dump(extcsv)
extcsv.add_data('FLIGHT_SUMMARY', flight_summary, field=flight_field)
#
# OZONE_REFERENCE -> O3Ref
ozoneref_field = 'Name, Model, Number, Version, TotalO3, WLCode, ObsType, UTC_Mean'
df_names = 'O3Ref_Name', 'O3Ref_Model', 'O3Ref_Number', 'O3Ref_Version', 'TotalO3_Col2A', 'WLCode', 'ObsType', 'UTC_Mean'
ozoneref_summary = make_summary(dfm,df_names)
extcsv.add_data('OZONE_REFERENCE', ozoneref_summary, field=ozoneref_field)
#
#
# PROFILE
data_names = 'Duration, Height, Pressure, Temperature, Humidity, TemperatureSonde, O3PartialPressure, SondeCurrent,O3PartialPressure_Uncertainty'
df_names = ['Time', 'Height','Pair', 'T', 'U', 'Tbox', 'O3', 'I', 'dO3']
size = len(df)
profile = [0] * size
for k in range(size):
profile[k] = df[df_names][k:k + 1].values[0]
profile[k] = ",".join([str(i) for i in profile[k] if str(i)])
extcsv.add_data('#PROFILE', profile[k], field= data_names)
out_name = path + str(df.at[df.first_valid_index(),'Date']) + '_testwoudc.csv'
print(out_name)
woudc_extcsv.dump(extcsv, out_name)
def test_dump_file(self):
"""Test file dumping"""
with self.assertRaises(TypeError):
dump()
def test_writer_reader(self):
"""
Produce woudc_extcsv.Writer object,
use woudc_extcsv.Reader to check values
"""
# produce extcsv object
extcsv = Writer(template=True)
# add data here
extcsv.add_comment('This file was generated by\
WODC_TO_CSX v1.0 using WODC 80-column formatted data.')
extcsv.add_comment('\'na\' is used where Instrument\
Model or Number are not available.')
extcsv.add_data('CONTENT', 'WOUDC,TotalOzone,1.0,1')
extcsv.add_data('DATA_GENERATION', '2014-08-28,NOAA-CMDL,0.0')
extcsv.add_data('PLATFORM', 'STN,031,MAUNA LOA,USA')
extcsv.add_data('INSTRUMENT', 'Dobson,Beck,076')
extcsv.add_data('LOCATION', '19.533,-155.574,3405')
extcsv.add_data('TIMESTAMP', '+00:00:00,2014-04-01')
extcsv.add_data('DAILY', '2014-04-01,0,2,283,,,,18',
field='Date,WLCode,ObsCode,ColumnO3,StdDevO3,\
UTC_Begin,UTC_End,UTC_Mean,nObs,mMu,ColumnSO2')
extcsv.add_data('DAILY', '2014-04-08,0,0,288,,,,23')
extcsv.add_data('DAILY', '2014-04-09,0,0,279,,,,23')
extcsv.add_data('DAILY', '2014-04-10,0,0,273,,,,24')
extcsv.add_data('DAILY', '2014-04-11,0,0,274,,,,21')
extcsv.add_data('DAILY', '2014-04-12,0,2,271,,,,18')
extcsv.add_data('DAILY', '2014-04-13,0,2,274,,,,18')
extcsv.add_data('DAILY', '2014-04-14,0,0,283,,,,23')
extcsv.add_data('DAILY', '2014-04-15,0,0,285,,,,23')
extcsv.add_data('DAILY', '2014-04-16,0,0,284,,,,23')
extcsv.add_data('DAILY', '2014-04-17,0,0,280,,,,22')
extcsv.add_data('DAILY', '2014-04-18,0,2,268,,,,18')
extcsv.add_data('DAILY', '2014-04-19,0,2,271,,,,18')
extcsv.add_data('DAILY', '2014-04-20,0,2,264,,,,18')
extcsv.add_data('DAILY', '2014-04-21,0,0,278,,,,23')
extcsv.add_data('DAILY', '2014-04-22,0,0,276,,,,21')
extcsv.add_data('DAILY', '2014-04-23,0,0,280,,,,23')
extcsv.add_data('DAILY', '2014-04-24,0,0,269,,,,22')
extcsv.add_data('DAILY', '2014-04-25,0,0,275,,,,21')
extcsv.add_data('DAILY', '2014-04-26,0,2,278,,,,18')
extcsv.add_data('DAILY', '2014-04-28,0,0,296,,,,21')
extcsv.add_data('DAILY', '2014-04-29,0,0,291,,,,23')
extcsv.add_data('DAILY', '2014-04-30,0,0,294,,,,21',
table_comment=' 1992 Coefficients in use')
extcsv.add_data('TIMESTAMP', '+00:00:00,2014-04-30',
field='UTCOffset,Date,Time', index=2)
extcsv.add_data('MONTHLY', '2014-04-01,279,8.3,23',
field='Date,ColumnO3,StdDevO3,Npts')
extcsv_s = dumps(extcsv)
# load my extcsv into Reader
my_extcsv_to = loads(extcsv_s)
# check tables
self.assertTrue('DAILY' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('PLATFORM' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('LOCATION' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('TIMESTAMP' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('DATA_GENERATION' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('TIMESTAMP2' in my_extcsv_to.sections,
'check totalozone table in my extcsv')
self.assertTrue('MONTHLY' in my_extcsv_to.sections,
'check totalozone in my extcsv')
# check fields
self.assertTrue('Level' in my_extcsv_to.sections['CONTENT'],
'check totalozone field in my extcsv')
self.assertTrue('UTCOffset' in my_extcsv_to.sections['TIMESTAMP'],
'check totalozone field in my extcsv')
self.assertTrue('ScientificAuthority'
in my_extcsv_to.sections['DATA_GENERATION'],
'check totalozone field in my extcsv')
self.assertTrue('Time' in my_extcsv_to.sections['TIMESTAMP2'],
'check totalozone field in my extcsv')
self.assertTrue('ColumnO3' in my_extcsv_to.sections['MONTHLY'],
'check totalozone field in my extcsv')
# check values
self.assertEqual('19.533',
my_extcsv_to.sections['LOCATION']['Latitude'],
'check totalozone value in my extcsv')
self.assertEqual('NOAA-CMDL',
my_extcsv_to.sections['DATA_GENERATION']['Agency'],
'check totalozone value in my extcsv')
self.assertEqual('1', my_extcsv_to.sections['CONTENT']['Form'],
'check totalozone value in my extcsv')
self.assertEqual('23', my_extcsv_to.sections['MONTHLY']['Npts'],
'check totalozone value in my extcsv')
my_to_daily = StringIO(my_extcsv_to.sections['DAILY']['_raw'])
my_daily_rows = csv.reader(my_to_daily)
my_daily_header = my_daily_rows.next()
self.assertTrue('WLCode' in my_daily_header,
'check totalozone daily field in my extcsv')
self.assertTrue('nObs' in my_daily_header,
'check totalozone daily field in my extcsv')
self.assertTrue('ColumnO3' in my_daily_header,
'check totalozone daily field in my extcsv')
self.assertTrue('ColumnSO2' in my_daily_header,
'check totalozone daily field in my extcsv')
self.assertEqual(0, my_daily_header.index('Date'),
'check totalozone daily field order in my extcsv')
self.assertEqual(len(my_daily_header) - 1,
my_daily_header.index('ColumnSO2'),
'check totalozone daily field order in my extcsv')
my_daily_row = None
# seek
for i in range(1, 6):
my_daily_row = my_daily_rows.next()
self.assertEqual('274',
my_daily_row[my_daily_header.index('ColumnO3')],
'check totalozone daily value in my extcsv')
self.assertEqual('',
my_daily_row[my_daily_header.index('StdDevO3')],
'check totalozone daily value in my extcsv')
self.assertEqual('',
my_daily_row[my_daily_header.index('UTC_Begin')],
'check totalozone daily value in my extcsv')
self.assertEqual('21',
my_daily_row[my_daily_header.index('UTC_Mean')],
'check totalozone daily value in my extcsv')
for i in range(1, 18):
my_daily_row = my_daily_rows.next()
self.assertEqual('291',
my_daily_row[my_daily_header.index('ColumnO3')],
'check totalozone daily value in my extcsv')
self.assertEqual('23',
my_daily_row[my_daily_header.index('UTC_Mean')],
'check totalozone daily value in my extcsv')
with self.assertRaises(TypeError):
extcsv_s = dump(extcsv)
extcsv.remove_data('CONTENT',
'Category',
data=10,
index=3,
all_occurences=True)
# clear file
# extcsv.clear_file()
extcsv.clear_table('CONTENT', index=3)
extcsv.add_data('CONTENT', 'new_value', index=3, field='Category')
extcsv.clear_field('CONTENT', index=3, field='Category')
# take a look at a table
extcsv.inspect_table('CONTENT', index=3)
# Add file level comments
extcsv.add_comment('Comment1')
extcsv.add_comment('Comment2')
extcsv.add_comment('Comment3')
'''
Write to file.
By default, the extcsv object will be validated for common/metadata
tables and fields. This file is missing some metadata tables and fields,
thus file will not serialize.
Violations will be printed to standard out and logged.
'''
woudc_extcsv.dump(extcsv, 'general-extcsv-example1.csv')
