def test_returns_self(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
filename = temp_fs.getsyspath("temp.h5")
_create_h5(filename)
reader = HDFReader(parent, filename)
self.assertEqual(reader, reader.__enter__())
def test_returns_self(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
reader = HDFReader(parent, filename)
self.assertEqual(reader, reader.__enter__())
def test_closes_reader(self, fake_close):
temp_fs = fsopendir("temp://")
parent = MagicMock()
filename = temp_fs.getsyspath("temp.h5")
_create_h5(filename)
reader = HDFReader(parent, filename)
reader.__exit__(None, None, None)
fake_close.assert_called_once_with()
def test_closes_reader(self, fake_close):
temp_fs = fsopendir('temp://')
parent = MagicMock()
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
reader = HDFReader(parent, filename)
reader.__exit__(None, None, None)
fake_close.assert_called_once_with()
def test_uses_cached_meta(self, fake_read):
temp_fs = fsopendir('temp://')
parent = MagicMock()
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
reader = HDFReader(parent, filename)
reader._meta = {}
reader.meta
fake_read.assert_not_called()
def test_uses_cached_meta(self, fake_read):
temp_fs = fsopendir("temp://")
parent = MagicMock()
filename = temp_fs.getsyspath("temp.h5")
_create_h5(filename)
reader = HDFReader(parent, filename)
reader._meta = {}
reader.meta
fake_read.assert_not_called()
def test_reads_meta_schema(self):
temp_fs = fsopendir('temp://')
filename = temp_fs.getsyspath('temp.h5')
# use minimal descriptor to make the test simplier.
descriptor = {
'pos': Int64Col(),
'name': StringCol(itemsize=255),
'type': StringCol(itemsize=255)
}
rows = [[('pos', float(i)), ('name', str(i)), ('type', str(i))]
for i in range(2)]
self._write_test_meta(temp_fs, 'schema', descriptor, rows)
with open_file(filename, mode='r') as h5_file:
ret = HDFReader._read_meta(h5_file)
self.assertIn('schema', ret)
self.assertEqual(len(ret['schema']),
3) # One for template, other for columns.
self.assertEqual(ret['schema'][0], MPRowsFile.SCHEMA_TEMPLATE)
self.assertEqual(len(ret['schema'][1]),
len(MPRowsFile.SCHEMA_TEMPLATE))
self.assertEqual(len(ret['schema'][0]),
len(MPRowsFile.SCHEMA_TEMPLATE))
pos_index = MPRowsFile.SCHEMA_TEMPLATE.index('pos')
name_index = MPRowsFile.SCHEMA_TEMPLATE.index('name')
self.assertEqual(ret['schema'][1][pos_index], 0)
self.assertEqual(ret['schema'][2][pos_index], 1.0)
self.assertEqual(ret['schema'][1][name_index], '0')
self.assertEqual(ret['schema'][2][name_index], '1')
def test_raises_ValueError_if_file_like_given(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
try:
HDFReader(parent, temp_fs.open('temp.h5', 'w'))
raise AssertionError('ValueError was not raised.')
except ValueError:
pass
def test_reads_meta_children(self, fake_child):
fake_child.return_value = {}
temp_fs = fsopendir("temp://")
filename = temp_fs.getsyspath("temp.h5")
_create_h5(filename)
with open_file(filename, mode="r") as h5_file:
HDFReader._read_meta(h5_file)
# _read_meta_child was called properly
self.assertEqual(len(fake_child.mock_calls), 7)
self.assertIn(call(h5_file, "about"), fake_child.mock_calls)
self.assertIn(call(h5_file, "excel"), fake_child.mock_calls)
self.assertIn(call(h5_file, "row_spec"), fake_child.mock_calls)
self.assertIn(call(h5_file, "source"), fake_child.mock_calls)
self.assertIn(call(h5_file, "comments"), fake_child.mock_calls)
self.assertIn(call(h5_file, "geo"), fake_child.mock_calls)
self.assertIn(call(h5_file, "schema"), fake_child.mock_calls)
def test_reads_meta_children(self, fake_child):
fake_child.return_value = {}
temp_fs = fsopendir('temp://')
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
with open_file(filename, mode='r') as h5_file:
HDFReader._read_meta(h5_file)
# _read_meta_child was called properly
self.assertEqual(len(fake_child.mock_calls), 7)
self.assertIn(call(h5_file, 'about'), fake_child.mock_calls)
self.assertIn(call(h5_file, 'excel'), fake_child.mock_calls)
self.assertIn(call(h5_file, 'row_spec'), fake_child.mock_calls)
self.assertIn(call(h5_file, 'source'), fake_child.mock_calls)
self.assertIn(call(h5_file, 'comments'), fake_child.mock_calls)
self.assertIn(call(h5_file, 'geo'), fake_child.mock_calls)
self.assertIn(call(h5_file, 'schema'), fake_child.mock_calls)
def test_reads_meta_if_cache_is_empty(self, fake_read):
temp_fs = fsopendir('temp://')
parent = MagicMock()
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
reader = HDFReader(parent, filename)
reader.meta
self.assertEqual(len(fake_read.mock_calls), 1)
def test_returns_default_template(self, fake_child):
fake_child.return_value = {}
temp_fs = fsopendir("temp://")
filename = temp_fs.getsyspath("temp.h5")
_create_h5(filename)
with open_file(filename, mode="r") as h5_file:
ret = HDFReader._read_meta(h5_file)
expected_keys = ["about", "excel", "row_spec", "source", "comments", "geo", "schema"]
self.assertEqual(sorted(expected_keys), sorted(ret.keys()))
def test_returns_default_template(self, fake_child):
fake_child.return_value = {}
temp_fs = fsopendir('temp://')
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
with open_file(filename, mode='r') as h5_file:
ret = HDFReader._read_meta(h5_file)
expected_keys = [
'about', 'excel', 'row_spec', 'source', 'comments', 'geo',
'schema'
]
self.assertEqual(sorted(expected_keys), sorted(ret.keys()))
def test_converts_hist_and_uvalues_json_to_list(self):
temp_fs = fsopendir("temp://")
# save meta.schema minimal table to the file.
descriptor = { # this is not valid descriptor, but I do not need it to be valid here.
"hist": StringCol(itemsize=255),
"uvalues": StringCol(itemsize=255),
}
self._write_test_meta(
temp_fs, "schema", descriptor, [[("hist", json.dumps([0, 1])), ("uvalues", json.dumps(["a", "b"]))]]
)
# now read it from file.
with open_file(temp_fs.getsyspath("temp.h5"), "r") as h5:
ret = HDFReader._read_meta_child(h5, "schema")
self.assertEqual(ret[0]["hist"], [0, 1])
self.assertEqual(ret[0]["uvalues"], ["a", "b"])
def test_reads_lines_to_dict(self):
temp_fs = fsopendir("temp://")
# save meta.about to the file.
descriptor = {"load_time": Float64Col(), "create_time": Float64Col()}
self._write_test_meta(temp_fs, "about", descriptor, [[("load_time", 1.0), ("create_time", 1.1)]])
# now read it from file.
with open_file(temp_fs.getsyspath("temp.h5"), "r") as h5:
ret = HDFReader._read_meta_child(h5, "about")
self.assertIsInstance(ret, list)
first = ret[0]
self.assertIn("load_time", first)
self.assertEqual(first["load_time"], 1.0)
self.assertIn("create_time", first)
self.assertEqual(first["create_time"], 1.1)
def test_converts_comment_rows_and_header_rows_json_to_list(self):
temp_fs = fsopendir('temp://')
# save meta.row_spec to the file.
descriptor = { # this is not valid descriptor, but I do not need it to be valid here.
'header_rows': StringCol(itemsize=255),
'comment_rows': StringCol(itemsize=255),
}
self._write_test_meta(temp_fs, 'row_spec', descriptor,
[[('comment_rows', json.dumps([0, 1])),
('header_rows', json.dumps([2, 3]))]])
# now read it from file.
with open_file(temp_fs.getsyspath('temp.h5'), 'r') as h5:
ret = HDFReader._read_meta_child(h5, 'row_spec')
self.assertEqual(ret[0]['comment_rows'], [0, 1])
self.assertEqual(ret[0]['header_rows'], [2, 3])
def test_converts_hist_and_uvalues_json_to_list(self):
temp_fs = fsopendir('temp://')
# save meta.schema minimal table to the file.
descriptor = { # this is not valid descriptor, but I do not need it to be valid here.
'hist': StringCol(itemsize=255),
'uvalues': StringCol(itemsize=255),
}
self._write_test_meta(temp_fs, 'schema', descriptor,
[[('hist', json.dumps([0, 1])),
('uvalues', json.dumps(['a', 'b']))]])
# now read it from file.
with open_file(temp_fs.getsyspath('temp.h5'), 'r') as h5:
ret = HDFReader._read_meta_child(h5, 'schema')
self.assertEqual(ret[0]['hist'], [0, 1])
self.assertEqual(ret[0]['uvalues'], ['a', 'b'])
def test_reads_lines_to_dict(self):
temp_fs = fsopendir('temp://')
# save meta.about to the file.
descriptor = {'load_time': Float64Col(), 'create_time': Float64Col()}
self._write_test_meta(temp_fs, 'about', descriptor,
[[('load_time', 1.0), ('create_time', 1.1)]])
# now read it from file.
with open_file(temp_fs.getsyspath('temp.h5'), 'r') as h5:
ret = HDFReader._read_meta_child(h5, 'about')
self.assertIsInstance(ret, list)
first = ret[0]
self.assertIn('load_time', first)
self.assertEqual(first['load_time'], 1.0)
self.assertIn('create_time', first)
self.assertEqual(first['create_time'], 1.1)
def test_contains_all_rows(self, fake_read):
fake_read.return_value = {}
temp_fs = fsopendir('temp://')
parent = MagicMock()
# save meta.about to the file.
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
# now read it from file.
reader = HDFReader(parent, filename)
raw_iter = reader.raw
first = next(raw_iter)
self.assertEqual(first, [0.0, 0.0, 0.0])
self.assertTrue(reader._in_iteration)
rows = list(raw_iter)
self.assertEqual(len(rows), 4)
self.assertFalse(reader._in_iteration)
def test_converts_comment_rows_and_header_rows_json_to_list(self):
temp_fs = fsopendir("temp://")
# save meta.row_spec to the file.
descriptor = { # this is not valid descriptor, but I do not need it to be valid here.
"header_rows": StringCol(itemsize=255),
"comment_rows": StringCol(itemsize=255),
}
self._write_test_meta(
temp_fs,
"row_spec",
descriptor,
[[("comment_rows", json.dumps([0, 1])), ("header_rows", json.dumps([2, 3]))]],
)
# now read it from file.
with open_file(temp_fs.getsyspath("temp.h5"), "r") as h5:
ret = HDFReader._read_meta_child(h5, "row_spec")
self.assertEqual(ret[0]["comment_rows"], [0, 1])
self.assertEqual(ret[0]["header_rows"], [2, 3])
def test_generates_rows_as_RowProxy_instances(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
reader = HDFReader(parent, filename)
with patch.object(HDFReader, 'headers',
['field1', 'field2', 'field3']):
rows_iter = iter(reader)
first = next(rows_iter)
self.assertIsInstance(first, RowProxy)
self.assertEqual(first.field1, 0.0)
self.assertEqual(first.field2, 0.0)
self.assertEqual(first.field3, 0.0)
self.assertTrue(reader._in_iteration)
rows = []
for row in rows_iter:
self.assertIsInstance(row, RowProxy)
rows.append(row)
self.assertEqual(len(rows), 4)
self.assertFalse(reader._in_iteration)
def test_reads_meta_schema(self):
temp_fs = fsopendir("temp://")
filename = temp_fs.getsyspath("temp.h5")
# use minimal descriptor to make the test simplier.
descriptor = {"pos": Int64Col(), "name": StringCol(itemsize=255), "type": StringCol(itemsize=255)}
rows = [[("pos", float(i)), ("name", str(i)), ("type", str(i))] for i in range(2)]
self._write_test_meta(temp_fs, "schema", descriptor, rows)
with open_file(filename, mode="r") as h5_file:
ret = HDFReader._read_meta(h5_file)
self.assertIn("schema", ret)
self.assertEqual(len(ret["schema"]), 3) # One for template, other for columns.
self.assertEqual(ret["schema"][0], MPRowsFile.SCHEMA_TEMPLATE)
self.assertEqual(len(ret["schema"][1]), len(MPRowsFile.SCHEMA_TEMPLATE))
self.assertEqual(len(ret["schema"][0]), len(MPRowsFile.SCHEMA_TEMPLATE))
pos_index = MPRowsFile.SCHEMA_TEMPLATE.index("pos")
name_index = MPRowsFile.SCHEMA_TEMPLATE.index("name")
self.assertEqual(ret["schema"][1][pos_index], 0)
self.assertEqual(ret["schema"][2][pos_index], 1.0)
self.assertEqual(ret["schema"][1][name_index], "0")
self.assertEqual(ret["schema"][2][name_index], "1")
