def test_writes_rows_is_cache_is_large(self, fake_write_rows):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.cache = [[] for i in range(10000)]
writer.insert_row(['row1'])
fake_write_rows.assert_called_once_with()
def test_writes_rows_is_cache_is_large(self, fake_write_rows):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.cache = [[] for i in range(10000)]
writer.insert_row(["row1"])
fake_write_rows.assert_called_once_with()
def test_inserts_and_writes_all_rows_from_source(self, fake_write_rows, fake_insert):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.load_rows([["row1"], ["row2"]])
fake_write_rows.assert_called_once_with()
self.assertEqual(fake_insert.mock_calls, [call(["row1"]), call(["row2"])])
def test_writes_meta(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, "meta")
def test_writes_meta(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, 'meta')
def test_inserts_row_to_the_cache(self, fake_write_rows):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.insert_row(['row1'])
self.assertEqual(writer.n_rows, 1)
self.assertEqual(writer.cache, [['row1']])
fake_write_rows.assert_not_called()
def test_inserts_row_to_the_cache(self, fake_write_rows):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.insert_row(["row1"])
self.assertEqual(writer.n_rows, 1)
self.assertEqual(writer.cache, [["row1"]])
fake_write_rows.assert_not_called()
def test_writes_rows_and_closes_file(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
h5_file = writer._h5_file
with patch.object(writer, '_write_rows') as fake_write:
writer.close()
fake_write.assert_called_once_with()
self.assertIsNone(writer._h5_file)
self.assertEqual(h5_file.isopen, 0)
def test_writes_rows_and_closes_file(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
h5_file = writer._h5_file
with patch.object(writer, "_write_rows") as fake_write:
writer.close()
fake_write.assert_called_once_with()
self.assertIsNone(writer._h5_file)
self.assertEqual(h5_file.isopen, 0)
def test_inserts_and_writes_all_rows_from_source(self, fake_write_rows,
fake_insert):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.load_rows([['row1'], ['row2']])
fake_write_rows.assert_called_once_with()
self.assertEqual(fake_insert.mock_calls,
[call(['row1']), call(['row2'])])
def test_writes_meta_geo(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.meta["geo"]["srs"] = 11
writer.meta["geo"]["bb"] = 22
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, "meta")
self.assertTrue(writer._h5_file.root.partition.meta, "geo")
self.assertEqual([(x["srs"], x["bb"]) for x in writer._h5_file.root.partition.meta.geo.iterrows()], [(11, 22)])
def test_writes_meta_row_spec(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.meta["row_spec"]["data_pattern"] = "pattern"
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, "meta")
self.assertTrue(writer._h5_file.root.partition.meta, "row_spec")
self.assertEqual(
[x["data_pattern"] for x in writer._h5_file.root.partition.meta.row_spec.iterrows()], [b("pattern")]
)
def test_writes_given_rows(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
# add two columns
writer.meta["schema"].append(self._get_column("col1", "int"))
writer.meta["schema"].append(self._get_column("col2", "str"))
writer._write_rows(rows=[[1, "row1"], [2, "row2"]])
# rows are written
self.assertEqual(writer._h5_file.root.partition.rows.nrows, 2)
self.assertEqual([x["col1"] for x in writer._h5_file.root.partition.rows.iterrows()], [1, 2])
self.assertEqual([x["col2"] for x in writer._h5_file.root.partition.rows.iterrows()], [b("row1"), b("row2")])
def test_saves_hist_and_uvalues_as_json_string(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer._validate_groups()
writer.meta["schema"].append(
self._get_column("col1", "str", predefined={"hist": [1, 2, 3], "uvalues": ["1", "2", "3"]})
)
writer._save_schema()
self.assertEqual(
[(x["hist"], x["uvalues"]) for x in writer._h5_file.root.partition.meta.schema.iterrows()],
[(b("[1, 2, 3]"), b('["1", "2", "3"]'))],
)
def test_writes_meta_row_spec(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.meta['row_spec']['data_pattern'] = 'pattern'
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, 'meta')
self.assertTrue(writer._h5_file.root.partition.meta, 'row_spec')
self.assertEqual([
x['data_pattern']
for x in writer._h5_file.root.partition.meta.row_spec.iterrows()
], [b('pattern')])
def test_writes_meta_source(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.meta["source"]["encoding"] = "utf-8"
writer.meta["source"]["url"] = "http://example.com"
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, "meta")
self.assertTrue(writer._h5_file.root.partition.meta, "source")
self.assertEqual(
[(x["encoding"], x["url"]) for x in writer._h5_file.root.partition.meta.source.iterrows()],
[(b("utf-8"), b("http://example.com"))],
)
def test_writes_meta_about(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.meta["about"]["load_time"] = 123.12
writer.meta["about"]["create_time"] = 111.11
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, "meta")
self.assertTrue(writer._h5_file.root.partition.meta, "about")
self.assertEqual(
[(x["load_time"], x["create_time"]) for x in writer._h5_file.root.partition.meta.about.iterrows()],
[(123.12, 111.11)],
)
def test_writes_meta_comments(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.meta["comments"]["header"] = "header"
writer.meta["comments"]["footer"] = "footer"
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, "meta")
self.assertTrue(writer._h5_file.root.partition.meta, "comments")
self.assertEqual(
[(x["header"], x["footer"]) for x in writer._h5_file.root.partition.meta.comments.iterrows()],
[(b("header"), b("footer"))],
)
def test_writes_meta_excel(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.meta["excel"]["worksheet"] = "sheet1"
writer.meta["excel"]["datemode"] = 1
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, "meta")
self.assertTrue(writer._h5_file.root.partition.meta, "excel")
self.assertEqual(
[(x["worksheet"], x["datemode"]) for x in writer._h5_file.root.partition.meta.excel.iterrows()],
[(b("sheet1"), 1)],
)
def test_raises_value_error_if_not_string_filename_given(self):
parent = MagicMock()
try:
HDFWriter(parent, MagicMock())
raise AssertionError('ValueError exception was not raised.')
except ValueError as exc:
self.assertIn('requires filename parameter as string', str(exc))
def test_writes_meta_schema(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer.meta["schema"].append(self._get_column("col1", "int"))
writer.meta["schema"].append(self._get_column("col2", "str"))
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, "meta")
self.assertTrue(writer._h5_file.root.partition.meta, "schema")
# check saved values.
saved = [(x["name"], x["type"]) for x in writer._h5_file.root.partition.meta.schema.iterrows()]
self.assertEqual(len(saved), len(writer.meta["schema"]) - 1)
self.assertEqual(saved, [(b("col1"), b("int")), (b("col2"), b("str"))])
def test_reads_meta_from_existing_file(self, fake_open, fake_read):
fake_read.return_value = {'about': {}}
temp_fs = fsopendir('temp://')
parent = MagicMock()
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
HDFWriter(parent, filename)
self.assertTrue(fake_read.called)
def test_opens_existing_file_in_append_mode(self, fake_open, fake_read):
fake_read.return_value = {'about': {}}
temp_fs = fsopendir('temp://')
parent = MagicMock()
filename = temp_fs.getsyspath('temp.h5')
_create_h5(filename)
HDFWriter(parent, filename)
self.assertIn(call(filename, mode='a'), fake_open.mock_calls)
def test_writes_given_rows(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
# add two columns
writer.meta['schema'].append(self._get_column('col1', 'int'))
writer.meta['schema'].append(self._get_column('col2', 'str'))
writer._write_rows(rows=[[1, 'row1'], [2, 'row2']])
# rows are written
self.assertEqual(writer._h5_file.root.partition.rows.nrows, 2)
self.assertEqual([
x['col1'] for x in writer._h5_file.root.partition.rows.iterrows()
], [1, 2])
self.assertEqual([
x['col2'] for x in writer._h5_file.root.partition.rows.iterrows()
], [b('row1'), b('row2')])
def test_writes_meta_schema(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.meta['schema'].append(self._get_column('col1', 'int'))
writer.meta['schema'].append(self._get_column('col2', 'str'))
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, 'meta')
self.assertTrue(writer._h5_file.root.partition.meta, 'schema')
# check saved values.
saved = [
(x['name'], x['type'])
for x in writer._h5_file.root.partition.meta.schema.iterrows()
]
self.assertEqual(len(saved), len(writer.meta['schema']) - 1)
self.assertEqual(saved, [(b('col1'), b('int')), (b('col2'), b('str'))])
def test_writes_file_header_to_table(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer._validate_groups()
writer.n_rows = 2
writer.n_cols = 3
writer.write_file_header()
self.assertIn('file_header', writer._h5_file.root.partition)
table = writer._h5_file.root.partition.file_header
self.assertEqual([(x['version'], x['n_rows'], x['n_cols'])
for x in table.iterrows()], [(1, 2, 3)])
def test_writes_file_header_to_table(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer._validate_groups()
writer.n_rows = 2
writer.n_cols = 3
writer.write_file_header()
self.assertIn("file_header", writer._h5_file.root.partition)
table = writer._h5_file.root.partition.file_header
self.assertEqual([(x["version"], x["n_rows"], x["n_cols"]) for x in table.iterrows()], [(1, 2, 3)])
def test_saves_header_rows_and_comment_rows_as_json_string(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer._validate_groups()
writer.meta['row_spec']['comment_rows'] = [0, 1]
writer.meta['row_spec']['header_rows'] = [2, 3]
descriptor = {
'end_row': Int32Col(),
'header_rows': StringCol(itemsize=255),
'start_row': Int32Col(),
'comment_rows': StringCol(itemsize=255),
'data_pattern': StringCol(itemsize=255)
}
writer._save_meta_child('row_spec', descriptor)
self.assertEqual(
[(x['comment_rows'], x['header_rows'])
for x in writer._h5_file.root.partition.meta.row_spec.iterrows()],
[(b('[0, 1]'), b('[2, 3]'))])
def test_writes_meta_about(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.meta['about']['load_time'] = 123.12
writer.meta['about']['create_time'] = 111.11
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, 'meta')
self.assertTrue(writer._h5_file.root.partition.meta, 'about')
self.assertEqual(
[(x['load_time'], x['create_time'])
for x in writer._h5_file.root.partition.meta.about.iterrows()],
[(123.12, 111.11)])
def test_writes_meta_comments(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.meta['comments']['header'] = 'header'
writer.meta['comments']['footer'] = 'footer'
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, 'meta')
self.assertTrue(writer._h5_file.root.partition.meta, 'comments')
self.assertEqual(
[(x['header'], x['footer'])
for x in writer._h5_file.root.partition.meta.comments.iterrows()],
[(b('header'), b('footer'))])
def test_writes_meta_excel(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.meta['excel']['worksheet'] = 'sheet1'
writer.meta['excel']['datemode'] = 1
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, 'meta')
self.assertTrue(writer._h5_file.root.partition.meta, 'excel')
self.assertEqual(
[(x['worksheet'], x['datemode'])
for x in writer._h5_file.root.partition.meta.excel.iterrows()],
[(b('sheet1'), 1)])
def test_writes_meta_geo(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.meta['geo']['srs'] = 11
writer.meta['geo']['bb'] = 22
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, 'meta')
self.assertTrue(writer._h5_file.root.partition.meta, 'geo')
self.assertEqual(
[(x['srs'], x['bb'])
for x in writer._h5_file.root.partition.meta.geo.iterrows()],
[(11, 22)])
def test_writes_meta_source(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer.meta['source']['encoding'] = 'utf-8'
writer.meta['source']['url'] = 'http://example.com'
writer._write_meta()
self.assertEqual(writer.cache, [])
self.assertTrue(writer._h5_file.root.partition, 'meta')
self.assertTrue(writer._h5_file.root.partition.meta, 'source')
self.assertEqual(
[(x['encoding'], x['url'])
for x in writer._h5_file.root.partition.meta.source.iterrows()],
[(b('utf-8'), b('http://example.com'))])
def test_saves_header_rows_and_comment_rows_as_json_string(self):
temp_fs = fsopendir("temp://")
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5"))
writer._validate_groups()
writer.meta["row_spec"]["comment_rows"] = [0, 1]
writer.meta["row_spec"]["header_rows"] = [2, 3]
descriptor = {
"end_row": Int32Col(),
"header_rows": StringCol(itemsize=255),
"start_row": Int32Col(),
"comment_rows": StringCol(itemsize=255),
"data_pattern": StringCol(itemsize=255),
}
writer._save_meta_child("row_spec", descriptor)
self.assertEqual(
[(x["comment_rows"], x["header_rows"]) for x in writer._h5_file.root.partition.meta.row_spec.iterrows()],
[(b("[0, 1]"), b("[2, 3]"))],
)
def test_saves_hist_and_uvalues_as_json_string(self):
temp_fs = fsopendir('temp://')
parent = MagicMock()
writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5'))
writer._validate_groups()
writer.meta['schema'].append(
self._get_column('col1',
'str',
predefined={
'hist': [1, 2, 3],
'uvalues': ['1', '2', '3']
}))
writer._save_schema()
self.assertEqual(
[(x['hist'], x['uvalues'])
for x in writer._h5_file.root.partition.meta.schema.iterrows()],
[(b('[1, 2, 3]'), b('["1", "2", "3"]'))])
