def test_writes_rows_is_cache_is_large(self, fake_write_rows): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.cache = [[] for i in range(10000)] writer.insert_row(['row1']) fake_write_rows.assert_called_once_with()
def test_writes_rows_is_cache_is_large(self, fake_write_rows): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.cache = [[] for i in range(10000)] writer.insert_row(["row1"]) fake_write_rows.assert_called_once_with()
def test_inserts_and_writes_all_rows_from_source(self, fake_write_rows, fake_insert): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.load_rows([["row1"], ["row2"]]) fake_write_rows.assert_called_once_with() self.assertEqual(fake_insert.mock_calls, [call(["row1"]), call(["row2"])])
def test_writes_meta(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, "meta")
def test_writes_meta(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, 'meta')
def test_inserts_row_to_the_cache(self, fake_write_rows): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.insert_row(['row1']) self.assertEqual(writer.n_rows, 1) self.assertEqual(writer.cache, [['row1']]) fake_write_rows.assert_not_called()
def test_inserts_row_to_the_cache(self, fake_write_rows): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.insert_row(["row1"]) self.assertEqual(writer.n_rows, 1) self.assertEqual(writer.cache, [["row1"]]) fake_write_rows.assert_not_called()
def test_writes_rows_and_closes_file(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) h5_file = writer._h5_file with patch.object(writer, '_write_rows') as fake_write: writer.close() fake_write.assert_called_once_with() self.assertIsNone(writer._h5_file) self.assertEqual(h5_file.isopen, 0)
def test_writes_rows_and_closes_file(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) h5_file = writer._h5_file with patch.object(writer, "_write_rows") as fake_write: writer.close() fake_write.assert_called_once_with() self.assertIsNone(writer._h5_file) self.assertEqual(h5_file.isopen, 0)
def test_inserts_and_writes_all_rows_from_source(self, fake_write_rows, fake_insert): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.load_rows([['row1'], ['row2']]) fake_write_rows.assert_called_once_with() self.assertEqual(fake_insert.mock_calls, [call(['row1']), call(['row2'])])
def test_writes_meta_geo(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.meta["geo"]["srs"] = 11 writer.meta["geo"]["bb"] = 22 writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, "meta") self.assertTrue(writer._h5_file.root.partition.meta, "geo") self.assertEqual([(x["srs"], x["bb"]) for x in writer._h5_file.root.partition.meta.geo.iterrows()], [(11, 22)])
def test_writes_meta_row_spec(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.meta["row_spec"]["data_pattern"] = "pattern" writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, "meta") self.assertTrue(writer._h5_file.root.partition.meta, "row_spec") self.assertEqual( [x["data_pattern"] for x in writer._h5_file.root.partition.meta.row_spec.iterrows()], [b("pattern")] )
def test_writes_given_rows(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) # add two columns writer.meta["schema"].append(self._get_column("col1", "int")) writer.meta["schema"].append(self._get_column("col2", "str")) writer._write_rows(rows=[[1, "row1"], [2, "row2"]]) # rows are written self.assertEqual(writer._h5_file.root.partition.rows.nrows, 2) self.assertEqual([x["col1"] for x in writer._h5_file.root.partition.rows.iterrows()], [1, 2]) self.assertEqual([x["col2"] for x in writer._h5_file.root.partition.rows.iterrows()], [b("row1"), b("row2")])
def test_saves_hist_and_uvalues_as_json_string(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer._validate_groups() writer.meta["schema"].append( self._get_column("col1", "str", predefined={"hist": [1, 2, 3], "uvalues": ["1", "2", "3"]}) ) writer._save_schema() self.assertEqual( [(x["hist"], x["uvalues"]) for x in writer._h5_file.root.partition.meta.schema.iterrows()], [(b("[1, 2, 3]"), b('["1", "2", "3"]'))], )
def test_writes_meta_row_spec(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.meta['row_spec']['data_pattern'] = 'pattern' writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, 'meta') self.assertTrue(writer._h5_file.root.partition.meta, 'row_spec') self.assertEqual([ x['data_pattern'] for x in writer._h5_file.root.partition.meta.row_spec.iterrows() ], [b('pattern')])
def test_writes_meta_source(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.meta["source"]["encoding"] = "utf-8" writer.meta["source"]["url"] = "http://example.com" writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, "meta") self.assertTrue(writer._h5_file.root.partition.meta, "source") self.assertEqual( [(x["encoding"], x["url"]) for x in writer._h5_file.root.partition.meta.source.iterrows()], [(b("utf-8"), b("http://example.com"))], )
def test_writes_meta_about(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.meta["about"]["load_time"] = 123.12 writer.meta["about"]["create_time"] = 111.11 writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, "meta") self.assertTrue(writer._h5_file.root.partition.meta, "about") self.assertEqual( [(x["load_time"], x["create_time"]) for x in writer._h5_file.root.partition.meta.about.iterrows()], [(123.12, 111.11)], )
def test_writes_meta_comments(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.meta["comments"]["header"] = "header" writer.meta["comments"]["footer"] = "footer" writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, "meta") self.assertTrue(writer._h5_file.root.partition.meta, "comments") self.assertEqual( [(x["header"], x["footer"]) for x in writer._h5_file.root.partition.meta.comments.iterrows()], [(b("header"), b("footer"))], )
def test_writes_meta_excel(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.meta["excel"]["worksheet"] = "sheet1" writer.meta["excel"]["datemode"] = 1 writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, "meta") self.assertTrue(writer._h5_file.root.partition.meta, "excel") self.assertEqual( [(x["worksheet"], x["datemode"]) for x in writer._h5_file.root.partition.meta.excel.iterrows()], [(b("sheet1"), 1)], )
def test_raises_value_error_if_not_string_filename_given(self): parent = MagicMock() try: HDFWriter(parent, MagicMock()) raise AssertionError('ValueError exception was not raised.') except ValueError as exc: self.assertIn('requires filename parameter as string', str(exc))
def test_writes_meta_schema(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer.meta["schema"].append(self._get_column("col1", "int")) writer.meta["schema"].append(self._get_column("col2", "str")) writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, "meta") self.assertTrue(writer._h5_file.root.partition.meta, "schema") # check saved values. saved = [(x["name"], x["type"]) for x in writer._h5_file.root.partition.meta.schema.iterrows()] self.assertEqual(len(saved), len(writer.meta["schema"]) - 1) self.assertEqual(saved, [(b("col1"), b("int")), (b("col2"), b("str"))])
def test_reads_meta_from_existing_file(self, fake_open, fake_read): fake_read.return_value = {'about': {}} temp_fs = fsopendir('temp://') parent = MagicMock() filename = temp_fs.getsyspath('temp.h5') _create_h5(filename) HDFWriter(parent, filename) self.assertTrue(fake_read.called)
def test_opens_existing_file_in_append_mode(self, fake_open, fake_read): fake_read.return_value = {'about': {}} temp_fs = fsopendir('temp://') parent = MagicMock() filename = temp_fs.getsyspath('temp.h5') _create_h5(filename) HDFWriter(parent, filename) self.assertIn(call(filename, mode='a'), fake_open.mock_calls)
def test_writes_given_rows(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) # add two columns writer.meta['schema'].append(self._get_column('col1', 'int')) writer.meta['schema'].append(self._get_column('col2', 'str')) writer._write_rows(rows=[[1, 'row1'], [2, 'row2']]) # rows are written self.assertEqual(writer._h5_file.root.partition.rows.nrows, 2) self.assertEqual([ x['col1'] for x in writer._h5_file.root.partition.rows.iterrows() ], [1, 2]) self.assertEqual([ x['col2'] for x in writer._h5_file.root.partition.rows.iterrows() ], [b('row1'), b('row2')])
def test_writes_meta_schema(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.meta['schema'].append(self._get_column('col1', 'int')) writer.meta['schema'].append(self._get_column('col2', 'str')) writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, 'meta') self.assertTrue(writer._h5_file.root.partition.meta, 'schema') # check saved values. saved = [ (x['name'], x['type']) for x in writer._h5_file.root.partition.meta.schema.iterrows() ] self.assertEqual(len(saved), len(writer.meta['schema']) - 1) self.assertEqual(saved, [(b('col1'), b('int')), (b('col2'), b('str'))])
def test_writes_file_header_to_table(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer._validate_groups() writer.n_rows = 2 writer.n_cols = 3 writer.write_file_header() self.assertIn('file_header', writer._h5_file.root.partition) table = writer._h5_file.root.partition.file_header self.assertEqual([(x['version'], x['n_rows'], x['n_cols']) for x in table.iterrows()], [(1, 2, 3)])
def test_writes_file_header_to_table(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer._validate_groups() writer.n_rows = 2 writer.n_cols = 3 writer.write_file_header() self.assertIn("file_header", writer._h5_file.root.partition) table = writer._h5_file.root.partition.file_header self.assertEqual([(x["version"], x["n_rows"], x["n_cols"]) for x in table.iterrows()], [(1, 2, 3)])
def test_saves_header_rows_and_comment_rows_as_json_string(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer._validate_groups() writer.meta['row_spec']['comment_rows'] = [0, 1] writer.meta['row_spec']['header_rows'] = [2, 3] descriptor = { 'end_row': Int32Col(), 'header_rows': StringCol(itemsize=255), 'start_row': Int32Col(), 'comment_rows': StringCol(itemsize=255), 'data_pattern': StringCol(itemsize=255) } writer._save_meta_child('row_spec', descriptor) self.assertEqual( [(x['comment_rows'], x['header_rows']) for x in writer._h5_file.root.partition.meta.row_spec.iterrows()], [(b('[0, 1]'), b('[2, 3]'))])
def test_writes_meta_about(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.meta['about']['load_time'] = 123.12 writer.meta['about']['create_time'] = 111.11 writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, 'meta') self.assertTrue(writer._h5_file.root.partition.meta, 'about') self.assertEqual( [(x['load_time'], x['create_time']) for x in writer._h5_file.root.partition.meta.about.iterrows()], [(123.12, 111.11)])
def test_writes_meta_comments(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.meta['comments']['header'] = 'header' writer.meta['comments']['footer'] = 'footer' writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, 'meta') self.assertTrue(writer._h5_file.root.partition.meta, 'comments') self.assertEqual( [(x['header'], x['footer']) for x in writer._h5_file.root.partition.meta.comments.iterrows()], [(b('header'), b('footer'))])
def test_writes_meta_excel(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.meta['excel']['worksheet'] = 'sheet1' writer.meta['excel']['datemode'] = 1 writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, 'meta') self.assertTrue(writer._h5_file.root.partition.meta, 'excel') self.assertEqual( [(x['worksheet'], x['datemode']) for x in writer._h5_file.root.partition.meta.excel.iterrows()], [(b('sheet1'), 1)])
def test_writes_meta_geo(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.meta['geo']['srs'] = 11 writer.meta['geo']['bb'] = 22 writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, 'meta') self.assertTrue(writer._h5_file.root.partition.meta, 'geo') self.assertEqual( [(x['srs'], x['bb']) for x in writer._h5_file.root.partition.meta.geo.iterrows()], [(11, 22)])
def test_writes_meta_source(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer.meta['source']['encoding'] = 'utf-8' writer.meta['source']['url'] = 'http://example.com' writer._write_meta() self.assertEqual(writer.cache, []) self.assertTrue(writer._h5_file.root.partition, 'meta') self.assertTrue(writer._h5_file.root.partition.meta, 'source') self.assertEqual( [(x['encoding'], x['url']) for x in writer._h5_file.root.partition.meta.source.iterrows()], [(b('utf-8'), b('http://example.com'))])
def test_saves_header_rows_and_comment_rows_as_json_string(self): temp_fs = fsopendir("temp://") parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath("temp.h5")) writer._validate_groups() writer.meta["row_spec"]["comment_rows"] = [0, 1] writer.meta["row_spec"]["header_rows"] = [2, 3] descriptor = { "end_row": Int32Col(), "header_rows": StringCol(itemsize=255), "start_row": Int32Col(), "comment_rows": StringCol(itemsize=255), "data_pattern": StringCol(itemsize=255), } writer._save_meta_child("row_spec", descriptor) self.assertEqual( [(x["comment_rows"], x["header_rows"]) for x in writer._h5_file.root.partition.meta.row_spec.iterrows()], [(b("[0, 1]"), b("[2, 3]"))], )
def test_saves_hist_and_uvalues_as_json_string(self): temp_fs = fsopendir('temp://') parent = MagicMock() writer = HDFWriter(parent, temp_fs.getsyspath('temp.h5')) writer._validate_groups() writer.meta['schema'].append( self._get_column('col1', 'str', predefined={ 'hist': [1, 2, 3], 'uvalues': ['1', '2', '3'] })) writer._save_schema() self.assertEqual( [(x['hist'], x['uvalues']) for x in writer._h5_file.root.partition.meta.schema.iterrows()], [(b('[1, 2, 3]'), b('["1", "2", "3"]'))])