class TestCTDNetCDFOceansites(unittest.TestCase):
def setUp(self):
self._infile = open(
os.path.join(
os.path.dirname(__file__),
'samples/nc_hyd/i08s_33RR20070204_00001_00001_hy1.nc'), 'r')
self.datafile = DataFile()
self._outfile = NamedTemporaryFile()
g = self.datafile.globals
g['DATE'] = '12341231'
g['TIME'] = '2359'
g['LATITUDE'] = 90
g['LONGITUDE'] = 180
g['DEPTH'] = -1
g['EXPOCODE'] = 'test'
g['STNNBR'] = '20'
g['CASTNO'] = '5'
g['_OS_ID'] = 'OS1'
fuse_datetime(self.datafile)
def tearDown(self):
self._infile.close()
def _setupData(self):
self.datafile['CTDPRS'] = Column('CTDPRS')
self.datafile['CTDPRS'].append(1, 2)
self.datafile['CTDOXY'] = Column('CTDOXY')
self.datafile['CTDOXY'].append(1, 2)
self.datafile.check_and_replace_parameters()
p = self.datafile['CTDOXY'].parameter
p.description = 'ctd oxygen'
p.bound_lower = 0
p.bound_upper = 200
def test_write(self):
self.assertRaises(AttributeError, ctdncos.write, self.datafile,
self._outfile)
self._setupData()
ctdncos.write(self.datafile, self._outfile)
def test_write_timeseries(self):
self._setupData()
ctdncos.write(self.datafile, self._outfile, timeseries='BATS')
def test_different_columns(self):
"""Columns between two datafiles differ under a wide variety of cases.
Case 1: Column values are different
Case 1 corollary: Flag values are different
Case 2: Units are different
Case 3: Column not in original
Case 4: Column not in derivative
"""
with TemporaryFile() as origin, TemporaryFile() as deriv:
origin.write("""\
BOTTLE,19700101CCHSIOYYY
# header 1
EXPOCODE,SECT_ID,STNNBR,CASTNO,SAMPNO,BTLNBR,BTLNBR_FLAG_W,LATITUDE,LONGITUDE,DATE,TIME,DEPTH,NITRAT,NITRAT_FLAG_W,NITRIT,DELC14,DELC14_FLAG_W
,,,,,,,,,,,METERS,UMOL/KG,,UMOL/KG,/MILLE,
316N145_9, TRNS1, 574, 1, 16, 36, 2, 0, 0, 19700101, 0000,1000,3.00,2,10.0,-999.000,9
316N145_9, TRNS1, 574, 1, 15, 35, 2, 0, 0, 19700101, 0000,1000,4.00,2,10.0,-999.000,9
END_DATA
""")
origin.flush()
origin.seek(0)
deriv.write("""\
BOTTLE,19700101CCHSIOYYY
# header 2
EXPOCODE,SECT_ID,STNNBR,CASTNO,SAMPNO,BTLNBR,BTLNBR_FLAG_W,LATITUDE,LONGITUDE,DATE,TIME,DEPTH,TDN,TDN_FLAG_W,NITRIT,DELC14,DELC14_FLAG_W,PH_SWS,PH_SWS_FLAG_W
,,,,,,,,,,,METERS,UMOL/KG,,NMOL/KG,/MILLE,,,
316N145_9, TRNS1, 574, 1, 16, 36, 2, 0, 0, 19700101, 0000,1000,6.00,3,10.0,-999.000,1,-999.0,9
316N145_9, TRNS1, 574, 1, 15, 35, 2, 0, 0, 19700101, 0000,1000,5.00,3,10.0, 10.000,9,-999.0,9
END_DATA
""")
deriv.flush()
deriv.seek(0)
dforigin = DataFile()
dfderiv = DataFile()
btlex.read(dforigin, origin)
btlex.read(dfderiv, deriv)
self.assertEqual(
# NITRIT comes after because NMOL/KG is not an expected unit and
# gets pushed to the end when sorting
(
['DELC14', 'DELC14_FLAG_W', 'NITRIT'],
# PH_SWS_FLAG_W has underscores inside the parameter name. All
# parts need to be included
['PH_SWS', 'PH_SWS_FLAG_W', 'TDN', 'TDN_FLAG_W'],
['NITRAT', 'NITRAT_FLAG_W'],
[
'EXPOCODE', 'SECT_ID', 'STNNBR', 'CASTNO', 'SAMPNO',
'BTLNBR', 'BTLNBR_FLAG_W', 'LATITUDE', 'LONGITUDE',
'DEPTH', '_DATETIME'
]),
different_columns(dforigin, dfderiv, (
'EXPOCODE',
'SECT_ID',
'STNNBR',
'CASTNO',
'SAMPNO',
'BTLNBR',
)))
lines = [
"DELC14 differs at origin row 1:\t(None, Decimal('10.000'))",
"DELC14_FLAG_W differs at origin row 0:\t(9, 1)",
]
self.assertTrue(self.ensure_lines(lines))
# Columns are not different if merged results are not different.
dfo = DataFile()
dfd = DataFile()
dfo.create_columns(['CTDPRS', 'CTDOXY'])
dfo.check_and_replace_parameters()
dfd.create_columns(['CTDPRS', 'CTDOXY'])
dfd.check_and_replace_parameters()
dfo['CTDPRS'].values = [1, 2, 3]
dfo['CTDOXY'].values = [10, 20, 30]
dfd['CTDPRS'].values = [3, 2, 1]
dfd['CTDOXY'].values = [30, 20, 10]
self.assertEqual(([], [], [], ['CTDPRS', 'CTDOXY']),
different_columns(dfo, dfd, ('CTDPRS', )))
class TestBottleNetCDF(unittest.TestCase):
def setUp(self):
self.infile = open(
sample_file('nc_hyd', 'i08s_33RR20070204_00001_00001_hy1.nc'), 'r')
def tearDown(self):
self.infile.close()
def assertAlmostEqualOrNones(self, x, y):
if x is None:
self.assert_(y is None)
else:
self.assertAlmostEqual(x, y)
def test_read(self):
self.file = DataFile()
botnc.read(self.file, self.infile)
nitrite_values = (0.11, None, 0.08, 0.08, 0.08, 0.08, 0.06, 0.03, 0.06,
0.04, 0.03, None, 0.03, None, 0.03, None)
map(lambda x: self.assertAlmostEqualOrNones(*x),
zip(nitrite_values, self.file.columns['NITRIT'].values))
freon11_values = (6.063, 6.055, 5.795, 5.619, 5.486, 5.508, 5.487,
5.683, 5.422, 5.190, 5.222, None, 5.289, None, 5.250,
5.254)
map(lambda x: self.assertAlmostEqualOrNones(*x),
zip(freon11_values, self.file.columns['CFC-11'].values))
freon113_values = (None, ) * 16
map(lambda x: self.assertAlmostEqualOrNones(*x),
zip(freon113_values, self.file.columns['CFC113'].values))
expocodes = ['33RR20070204'] * 16
self.assertEqual(expocodes, self.file.columns['EXPOCODE'].values)
def test_read_multiple(self):
self.file = DataFile()
botnc.read(self.file, self.infile)
nitrite_values = (0.11, None, 0.08, 0.08, 0.08, 0.08, 0.06, 0.03, 0.06,
0.04, 0.03, None, 0.03, None, 0.03, None)
map(lambda x: self.assertAlmostEqualOrNones(*x),
zip(nitrite_values, self.file.columns['NITRIT'].values))
freon11_values = (6.063, 6.055, 5.795, 5.619, 5.486, 5.508, 5.487,
5.683, 5.422, 5.190, 5.222, None, 5.289, None, 5.250,
5.254)
map(lambda x: self.assertAlmostEqualOrNones(*x),
zip(freon11_values, self.file.columns['CFC-11'].values))
freon113_values = (None, ) * 16
map(lambda x: self.assertAlmostEqualOrNones(*x),
zip(freon113_values, self.file.columns['CFC113'].values))
expocodes = ['33RR20070204'] * 16
self.assertEqual(expocodes, self.file.columns['EXPOCODE'].values)
# Read second file
infile2 = open(sample_file('nc_hyd', 'p03a_00199_00001_hy1.nc'), 'r')
botnc.read(self.file, infile2)
# Make sure all columns have the same length
length = None
for c in self.file.columns.values():
if not length:
length = len(c.values)
else:
self.assertEquals(len(c.values), length)
if c.is_flagged_woce():
self.assertEquals(len(c.flags_woce), length)
if c.is_flagged_igoss():
self.assertEquals(len(c.flags_igoss), length)
# Test parameter in first file not in second is filled with None.
freon113_values += (None, ) * 36
map(lambda x: self.assertAlmostEqualOrNones(*x),
zip(freon113_values, self.file.columns['CFC113'].values))
# Test parameter in both files are filled in correctly.
freon11_values += (1.437, 1.501, 1.515, 1.525, 1.578, 1.596, 1.602,
1.725, 1.650, 1.703, 1.694, 1.437, 1.059, 0.702,
0.303, 0.130, 0.040, 0.015, -0.001, 0.002, 0.000,
None, None, 0.012, None, 0.006, None, None, None,
0.014, None, 0.000, None, 0.014, None, -0.001)
map(lambda x: self.assertAlmostEqualOrNones(*x),
zip(freon11_values, self.file.columns['CFC-11'].values))
infile2.close()
def test_write(self):
self.file = DataFile()
g = self.file.globals
self.file['EXPOCODE'] = Column('EXPOCODE')
self.file['EXPOCODE'].append('TESTEXPO')
self.file['SECT_ID'] = Column('SECT_ID')
self.file['SECT_ID'].append('TEST')
self.file['STNNBR'] = Column('CASTNO')
self.file['STNNBR'].append(5)
self.file['CASTNO'] = Column('STNNBR')
self.file['CASTNO'].append(20)
self.file['DEPTH'] = Column('DEPTH')
self.file['DEPTH'].append(-1)
self.file['LATITUDE'] = Column('LATITUDE')
self.file['LATITUDE'].append(90)
self.file['LONGITUDE'] = Column('LONGITUDE')
self.file['LONGITUDE'].append(180)
self.file['_DATETIME'] = Column('_DATETIME')
self.file['_DATETIME'].append(datetime.utcnow())
self.file['BTLNBR'] = Column('BTLNBR')
self.file['BTLNBR'].append(5, 9)
self.file['CTDOXY'] = Column('CTDOXY')
self.file['CTDOXY'].append(1, 2)
self.file.check_and_replace_parameters()
p = self.file['CTDOXY'].parameter
p.description = 'ctd oxygen'
p.bound_lower = 0
p.bound_upper = 200
botnc.write(self.file, NamedTemporaryFile())
class TestDataFile(TestCase):
def setUp(self):
self.file = DataFile()
self.c = self.file.columns['EXPOCODE'] = Column('EXPOCODE')
def tearDown(self):
self.file = None
def test_init(self):
self.assertEqual(len(self.file.columns), 1)
self.assertEqual(self.file.footer, None)
self.assertEqual(self.file.globals, {'stamp': '', 'header': ''})
def test_expocodes(self):
self.c.append('A')
self.assertEqual(['A'], self.file.expocodes())
self.c.append('B')
self.assertEqual(['A', 'B'], self.file.expocodes())
self.c.append('A')
self.assertEqual(
['A', 'B'],
self.file.expocodes()) # Expocodes returns unique expocodes.
def test_len(self):
c = self.file.columns['EXPOCODE']
del self.file.columns['EXPOCODE']
self.assertEqual(len(self.file), 0)
self.file.columns['EXPOCODE'] = c
self.assertEqual(len(self.file), 0)
self.c.append('A')
self.assertEqual(len(self.file), 1)
self.c.append('A')
self.assertEqual(len(self.file), 2)
def test_sorted_columns(self):
self.file.columns['CASTNO'] = Column('CASTNO')
self.file.columns['STNNBR'] = Column('STNNBR')
expected = ['EXPOCODE', 'STNNBR', 'CASTNO']
received = map(lambda c: c.parameter.mnemonic_woce(),
self.file.sorted_columns())
# If lengths are equal and all expected in received, then assume equal
self.assertEqual(len(expected), len(received))
self.assertTrue(all([x in received for x in expected]))
def test_get_property_for_columns(self):
pass # This is tested by the following tests.
def test_column_headers(self):
self.assertEqual(['EXPOCODE'], self.file.column_headers())
self.file.columns['STNNBR'] = Column('STNNBR')
expected = ['EXPOCODE', 'STNNBR']
received = self.file.column_headers()
# If lengths are equal and all expected in received, then assume equal
self.assertEqual(len(expected), len(received))
self.assertTrue(all([x in received for x in expected]))
def test_formats(self):
self.file.columns['CTDOXY'] = Column('CTDOXY')
self.file.check_and_replace_parameters()
# Order of columns may be wrong
self.assertEqual(['%11s', '%9.4f'], self.file.formats())
def test_to_dict(self):
self.file.to_dict()
pass # TODO
def test_str(self):
str(self.file)
def test_create_columns(self):
parameters = ['CTDOXY']
units = ['UMOL/KG']
self.file.create_columns(parameters, units)
def test_column_append(self):
self.assertEqual(self.c.values, [])
self.c.set(2, 'test')
self.assertEqual(self.c.values, [None, None, 'test'])
self.assertEqual(self.c.flags_woce, [])
self.c.append('test2', 'flag2')
self.assertEqual(self.c.values, [None, None, 'test', 'test2'])
self.assertEqual(self.c.flags_woce, [None, None, None, 'flag2'])
def test_calculate_depths(self):
self.file['_ACTUAL_DEPTH'] = Column('_ACTUAL_DEPTH')
self.assertEqual(('actual', []), self.file.calculate_depths())
del self.file['_ACTUAL_DEPTH']
self.file.globals['LATITUDE'] = 0
self.file.create_columns(['CTDPRS', 'CTDSAL', 'CTDTMP'])
self.assertEqual(('unesco1983', []), self.file.calculate_depths())
self.file['CTDPRS'].values = [1]
self.file['CTDSAL'].values = [1]
self.file['CTDTMP'].values = [1]
self.assertEqual(
('sverdrup', [_decimal('1.021723814950101286444879340E-8')]),
self.file.calculate_depths())
def test_check_and_replace_parameter_contrived(self):
"""Contrived parameters are not checked."""
col = Column('_DATETIME')
col.check_and_replace_parameter(self.file, convert=False)
def read(self, handle, metadata=None):
"""How to read a Bottle Bermuda Atlantic Time-Series Study file.
This function reads bats_bottle.txt.
Arguments:
self - (special case, see NOTE) dictionary
metadata - (optional) BATS cruise metadata to be used to find port dates
NOTE: The result for this method is a special case. The bottle file format
contains the entire BATS holdings while the internal data format splits data
up by cruises. Because cruises for timeseries are split by file for cruise,
the end result is a dictionary with cruise_ids as keys to
DatafileCollections (cruises) containing Datafiles (casts).
"""
sections = _read_header_sections(self, handle)
_read_variables(self, handle)
parameters = _get_variables(self, handle, sections)
# Add DON for note in Variables list stating DON is reported for TON prior
# to BATS 121
parameters.append(['DON', None, 'umol/kg'])
manual_parameters = [
['BTLNBR', ''],
['_DATETIME', ''],
['LATITUDE', ''],
['LONGITUDE', ''],
['_ACTUAL_DEPTH', 'METERS'],
]
columns = [x[0] for x in manual_parameters]
units = [x[1] for x in manual_parameters]
s = None
for i, (var, d, u) in enumerate(parameters):
if var == 'Depth':
s = i + 1
continue
# Only want to add parameters after Depth. The others were done manually.
if s is None:
continue
try:
var = bats_to_param[var]
except KeyError:
pass
columns.append(var)
units.append(u)
template_df = DataFile()
template_df.create_columns(columns, units)
template_df.check_and_replace_parameters(convert=False)
for sec, lines in sections.items():
if sec == 'Variables list':
continue
if sec != 'Comments':
continue
template_df.globals['_{0}'.format(sec)] = '\n'.join(lines)
df = None
params_auto = parameters[s:]
dfi = 0
for i, l in enumerate(handle):
parts = l.split()
id = parts[0]
(cruise_type, type_id, cruise_num, cruise_id, cast_type, cast_id,
nisk_id) = _parse_bats_id(id)
ship = _ship_from_cruise_num(cruise_num)
if not ship:
ship = 'R/V Atlantic Explorer'
if (df is None or df.globals['_OS_ID'] != cruise_id
or df.globals['STNNBR'] != cruise_type
or df.globals['CASTNO'] != cast_id):
if df is not None:
# Done reading one cast. Finalize it.
log.info(u'finalizing cast {0} {1} {2}'.format(
df.globals['_OS_ID'], df.globals['STNNBR'],
df.globals['CASTNO']))
try:
meta = metadata[cruise_id]
port_date = meta['dates'][0]
except (TypeError, KeyError):
port_date = None
if not port_date:
port_date = min(df['_DATETIME'])
df.globals['EXPOCODE'] = create_expocode(
ship_code(ship, raise_on_unknown=False), port_date)
log.info(df.globals['EXPOCODE'])
df.globals['DEPTH'] = max(df['_ACTUAL_DEPTH'])
collapse_globals(df, ['_DATETIME', 'LATITUDE', 'LONGITUDE'])
# Normalize all the parameter column lengths. There may be
# columns that did not get data written to them so make sure
# they are just as long as the rest
length = len(df)
for c in df.columns.values():
c.set_length(length)
try:
dfc = self[df.globals['_OS_ID']]
except KeyError:
dfc = self[df.globals['_OS_ID']] = DataFileCollection()
dfc.files.append(df)
dfi = 0
# Create a new cast
df = copy(template_df)
df.globals['SECT_ID'] = BATS_SECT_ID
df.globals['_SHIP'] = ship
df.globals['_OS_ID'] = cruise_id
df.globals['STNNBR'] = cruise_type
df.globals['CASTNO'] = cast_id
df['BTLNBR'].set(dfi, nisk_id)
dt_ascii = datetime.strptime(parts[1] + parts[3], '%Y%m%d%H%M')
dt_deci = bats_time_to_dt(parts[2])
#if dt_ascii != dt_deci:
# log.warn(
# u'Dates differ on data row {0}: {5} {1!r}={2} '
# '{3!r}={4}'.format(i, parts[1] + parts[3], dt_ascii, parts[2],
# dt_deci, dt_deci - dt_ascii))
df['_DATETIME'].set(dfi, dt_ascii)
df['LATITUDE'].set(dfi, Decimal(parts[4]))
df['LONGITUDE'].set(dfi, Decimal(correct_longitude(parts[5])))
df['_ACTUAL_DEPTH'].set_check_range(dfi, Decimal(parts[6]))
parts_auto = parts[s:]
for p, v in zip(params_auto, parts_auto):
param = p[0]
try:
param = bats_to_param[param]
except KeyError:
pass
if cruise_num < 121 and param == 'TON':
param = 'DON'
if (equal_with_epsilon(v, -9) or equal_with_epsilon(v, -9.9)
or equal_with_epsilon(v, -9.99)):
df[param].set_check_range(dfi, None)
# TODO determine whether -10 is just bad formatting for -9.9
elif equal_with_epsilon(v, -10):
#log.warn(u'Possible missing data value {0}'.format(v))
df[param].set_check_range(dfi, None)
elif v == 0:
log.warn(u'Data under detection limit, set flag to '
'WOCE water sample questionable measurement')
df[param].set_check_range(dfi, None, flag=3)
else:
df[param].set_check_range(dfi, Decimal(v))
dfi += 1
# Since this is a super long file that contains multiple cruises and
# casts, as the file is processed it is split apart into a list of
# DataFileCollection(s) containing DataFile objects for each casts
if i % 100 == 0:
log.info(u'processed {0} lines'.format(i))