def test_write_default_config(self): import tempfile tdir = tempfile.mkdtemp() outpth = os.path.join(tdir, 'defconfig.yml') SimpleDelimitedParser.write_default_parser_config(outpth) parser = SimpleDelimitedParser.get_parser( 'test_data/testmanualupload.csv', outpth) props = parser.properties want_props = { 'data_url': 'test_data/testmanualupload.csv', 'header_size': 0, 'delimiter': ',', 'num_columns': None, 'use_column_names': True, 'dtype': 'float32', 'fill_val': -999, 'column_map': None } for p in props.keys(): self.assertEqual(props[p], want_props[p]) import shutil shutil.rmtree(outpth, ignore_errors=True)
def test_parse_no_names(self): parser = SimpleDelimitedParser.get_parser('test_data/testmanualupload.csv', 'test_data/testmanualupload.yml') parser.use_column_names = False parser.header_size = 1 props = parser.properties want_props = { 'data_url': 'test_data/testmanualupload.csv', 'header_size': 1, 'delimiter': ',', 'num_columns': 3, 'use_column_names': False, 'dtype': 'int8', 'fill_val': 0, 'column_map': OrderedDict([(0, OrderedDict([('name', 'time'), ('dtype', 'int64'), ('fill_val', -9)]))]) } for p in props.keys(): self.assertEqual(props[p], want_props[p]) dat = parser.parse() self.assertEqual(dat.dtype.names, ('time', 'f1', 'f2')) want_vals = { 'time': np.arange(10, dtype='int64'), 'f1': np.array([0, 0, 0, 0, 1, 0, 0, 1, 0, 0], dtype=bool), 'f2': np.array([1, 0, 1, 0, 0, 0, 1, 1, 0, 0], dtype=bool) } for x in dat.dtype.names: np.testing.assert_array_equal(dat[x], want_vals[x])
def test_parse_no_config(self): parser = SimpleDelimitedParser.get_parser('test_data/testmanualupload.csv') props = parser.properties want_props = { 'data_url': 'test_data/testmanualupload.csv', 'header_size': 0, 'delimiter': ',', 'num_columns': None, 'use_column_names': True, 'dtype': 'float32', 'fill_val': -999, 'column_map': None } for p in props.keys(): self.assertEqual(props[p], want_props[p]) dat = parser.parse() self.assertEqual(dat.dtype.names, ('time', 'temp_hitl_qc', 'cond_hitl_qc')) want_vals = { 'time': np.arange(10, dtype='int64'), 'temp_hitl_qc': np.array([0, 0, 0, 0, 1, 0, 0, 1, 0, 0], dtype=bool), 'cond_hitl_qc': np.array([1, 0, 1, 0, 0, 0, 1, 1, 0, 0], dtype=bool) } for x in dat.dtype.names: np.testing.assert_array_equal(dat[x], want_vals[x])
def test_write_default_config(self): import tempfile tdir = tempfile.mkdtemp() outpth = os.path.join(tdir, 'defconfig.yml') SimpleDelimitedParser.write_default_parser_config(outpth) parser = SimpleDelimitedParser.get_parser('test_data/testmanualupload.csv', outpth) props = parser.properties want_props = { 'data_url': 'test_data/testmanualupload.csv', 'header_size': 0, 'delimiter': ',', 'num_columns': None, 'use_column_names': True, 'dtype': 'float32', 'fill_val': -999, 'column_map': None } for p in props.keys(): self.assertEqual(props[p], want_props[p]) import shutil shutil.rmtree(outpth, ignore_errors=True)