def test_write_default_config(self):
import tempfile
tdir = tempfile.mkdtemp()
outpth = os.path.join(tdir, 'defconfig.yml')
SimpleDelimitedParser.write_default_parser_config(outpth)
parser = SimpleDelimitedParser.get_parser(
'test_data/testmanualupload.csv', outpth)
props = parser.properties
want_props = {
'data_url': 'test_data/testmanualupload.csv',
'header_size': 0,
'delimiter': ',',
'num_columns': None,
'use_column_names': True,
'dtype': 'float32',
'fill_val': -999,
'column_map': None
}
for p in props.keys():
self.assertEqual(props[p], want_props[p])
import shutil
shutil.rmtree(outpth, ignore_errors=True)
def test_parse_no_names(self):
parser = SimpleDelimitedParser.get_parser('test_data/testmanualupload.csv', 'test_data/testmanualupload.yml')
parser.use_column_names = False
parser.header_size = 1
props = parser.properties
want_props = {
'data_url': 'test_data/testmanualupload.csv',
'header_size': 1,
'delimiter': ',',
'num_columns': 3,
'use_column_names': False,
'dtype': 'int8',
'fill_val': 0,
'column_map': OrderedDict([(0, OrderedDict([('name', 'time'), ('dtype', 'int64'), ('fill_val', -9)]))])
}
for p in props.keys():
self.assertEqual(props[p], want_props[p])
dat = parser.parse()
self.assertEqual(dat.dtype.names, ('time', 'f1', 'f2'))
want_vals = {
'time': np.arange(10, dtype='int64'),
'f1': np.array([0, 0, 0, 0, 1, 0, 0, 1, 0, 0], dtype=bool),
'f2': np.array([1, 0, 1, 0, 0, 0, 1, 1, 0, 0], dtype=bool)
}
for x in dat.dtype.names:
np.testing.assert_array_equal(dat[x], want_vals[x])
def test_parse_no_config(self):
parser = SimpleDelimitedParser.get_parser('test_data/testmanualupload.csv')
props = parser.properties
want_props = {
'data_url': 'test_data/testmanualupload.csv',
'header_size': 0,
'delimiter': ',',
'num_columns': None,
'use_column_names': True,
'dtype': 'float32',
'fill_val': -999,
'column_map': None
}
for p in props.keys():
self.assertEqual(props[p], want_props[p])
dat = parser.parse()
self.assertEqual(dat.dtype.names, ('time', 'temp_hitl_qc', 'cond_hitl_qc'))
want_vals = {
'time': np.arange(10, dtype='int64'),
'temp_hitl_qc': np.array([0, 0, 0, 0, 1, 0, 0, 1, 0, 0], dtype=bool),
'cond_hitl_qc': np.array([1, 0, 1, 0, 0, 0, 1, 1, 0, 0], dtype=bool)
}
for x in dat.dtype.names:
np.testing.assert_array_equal(dat[x], want_vals[x])
def test_parse_no_names(self):
parser = SimpleDelimitedParser.get_parser('test_data/testmanualupload.csv', 'test_data/testmanualupload.yml')
parser.use_column_names = False
parser.header_size = 1
props = parser.properties
want_props = {
'data_url': 'test_data/testmanualupload.csv',
'header_size': 1,
'delimiter': ',',
'num_columns': 3,
'use_column_names': False,
'dtype': 'int8',
'fill_val': 0,
'column_map': OrderedDict([(0, OrderedDict([('name', 'time'), ('dtype', 'int64'), ('fill_val', -9)]))])
}
for p in props.keys():
self.assertEqual(props[p], want_props[p])
dat = parser.parse()
self.assertEqual(dat.dtype.names, ('time', 'f1', 'f2'))
want_vals = {
'time': np.arange(10, dtype='int64'),
'f1': np.array([0, 0, 0, 0, 1, 0, 0, 1, 0, 0], dtype=bool),
'f2': np.array([1, 0, 1, 0, 0, 0, 1, 1, 0, 0], dtype=bool)
}
for x in dat.dtype.names:
np.testing.assert_array_equal(dat[x], want_vals[x])
def test_parse_no_config(self):
parser = SimpleDelimitedParser.get_parser('test_data/testmanualupload.csv')
props = parser.properties
want_props = {
'data_url': 'test_data/testmanualupload.csv',
'header_size': 0,
'delimiter': ',',
'num_columns': None,
'use_column_names': True,
'dtype': 'float32',
'fill_val': -999,
'column_map': None
}
for p in props.keys():
self.assertEqual(props[p], want_props[p])
dat = parser.parse()
self.assertEqual(dat.dtype.names, ('time', 'temp_hitl_qc', 'cond_hitl_qc'))
want_vals = {
'time': np.arange(10, dtype='int64'),
'temp_hitl_qc': np.array([0, 0, 0, 0, 1, 0, 0, 1, 0, 0], dtype=bool),
'cond_hitl_qc': np.array([1, 0, 1, 0, 0, 0, 1, 1, 0, 0], dtype=bool)
}
for x in dat.dtype.names:
np.testing.assert_array_equal(dat[x], want_vals[x])
def test_write_default_config(self):
import tempfile
tdir = tempfile.mkdtemp()
outpth = os.path.join(tdir, 'defconfig.yml')
SimpleDelimitedParser.write_default_parser_config(outpth)
parser = SimpleDelimitedParser.get_parser('test_data/testmanualupload.csv', outpth)
props = parser.properties
want_props = {
'data_url': 'test_data/testmanualupload.csv',
'header_size': 0,
'delimiter': ',',
'num_columns': None,
'use_column_names': True,
'dtype': 'float32',
'fill_val': -999,
'column_map': None
}
for p in props.keys():
self.assertEqual(props[p], want_props[p])
import shutil
shutil.rmtree(outpth, ignore_errors=True)
