class HDFStoreDataFrame(BaseIO):
def setup(self):
N = 25000
index = tm.makeStringIndex(N)
self.df = DataFrame({'float1': np.random.randn(N),
'float2': np.random.randn(N)},
index=index)
self.df_mixed = DataFrame({'float1': np.random.randn(N),
'float2': np.random.randn(N),
'string1': ['foo'] * N,
'bool1': [True] * N,
'int1': np.random.randint(0, N, size=N)},
index=index)
self.df_wide = DataFrame(np.random.randn(N, 100))
self.start_wide = self.df_wide.index[10000]
self.stop_wide = self.df_wide.index[15000]
self.df2 = DataFrame({'float1': np.random.randn(N),
'float2': np.random.randn(N)},
index=date_range('1/1/2000', periods=N))
self.start = self.df2.index[10000]
self.stop = self.df2.index[15000]
self.df_wide2 = DataFrame(np.random.randn(N, 100),
index=date_range('1/1/2000', periods=N))
self.df_dc = DataFrame(np.random.randn(N, 10),
columns=['C%03d' % i for i in range(10)])
self.fname = '__test__.h5'
self.store = HDFStore(self.fname)
self.store.put('fixed', self.df)
self.store.put('fixed_mixed', self.df_mixed)
self.store.append('table', self.df2)
self.store.append('table_mixed', self.df_mixed)
self.store.append('table_wide', self.df_wide)
self.store.append('table_wide2', self.df_wide2)
def teardown(self):
self.store.close()
self.remove(self.fname)
def time_read_store(self):
self.store.get('fixed')
def time_read_store_mixed(self):
self.store.get('fixed_mixed')
def time_write_store(self):
self.store.put('fixed_write', self.df)
def time_write_store_mixed(self):
self.store.put('fixed_mixed_write', self.df_mixed)
def time_read_store_table_mixed(self):
self.store.select('table_mixed')
def time_write_store_table_mixed(self):
self.store.append('table_mixed_write', self.df_mixed)
def time_read_store_table(self):
self.store.select('table')
def time_write_store_table(self):
self.store.append('table_write', self.df)
def time_read_store_table_wide(self):
self.store.select('table_wide')
def time_write_store_table_wide(self):
self.store.append('table_wide_write', self.df_wide)
def time_write_store_table_dc(self):
self.store.append('table_dc_write', self.df_dc, data_columns=True)
def time_query_store_table_wide(self):
self.store.select('table_wide', where="index > self.start_wide and "
"index < self.stop_wide")
def time_query_store_table(self):
self.store.select('table', where="index > self.start and "
"index < self.stop")
def time_store_repr(self):
repr(self.store)
def time_store_str(self):
str(self.store)
def time_store_info(self):
self.store.info()
def test_multiple_open_close(setup_path):
# gh-4409: open & close multiple times
with ensure_clean_path(setup_path) as path:
df = tm.makeDataFrame()
df.to_hdf(path, "df", mode="w", format="table")
# single
store = HDFStore(path)
assert "CLOSED" not in store.info()
assert store.is_open
store.close()
assert "CLOSED" in store.info()
assert not store.is_open
with ensure_clean_path(setup_path) as path:
if pytables._table_file_open_policy_is_strict:
# multiples
store1 = HDFStore(path)
msg = (
r"The file [\S]* is already opened\. Please close it before "
r"reopening in write mode\."
)
with pytest.raises(ValueError, match=msg):
HDFStore(path)
store1.close()
else:
# multiples
store1 = HDFStore(path)
store2 = HDFStore(path)
assert "CLOSED" not in store1.info()
assert "CLOSED" not in store2.info()
assert store1.is_open
assert store2.is_open
store1.close()
assert "CLOSED" in store1.info()
assert not store1.is_open
assert "CLOSED" not in store2.info()
assert store2.is_open
store2.close()
assert "CLOSED" in store1.info()
assert "CLOSED" in store2.info()
assert not store1.is_open
assert not store2.is_open
# nested close
store = HDFStore(path, mode="w")
store.append("df", df)
store2 = HDFStore(path)
store2.append("df2", df)
store2.close()
assert "CLOSED" in store2.info()
assert not store2.is_open
store.close()
assert "CLOSED" in store.info()
assert not store.is_open
# double closing
store = HDFStore(path, mode="w")
store.append("df", df)
store2 = HDFStore(path)
store.close()
assert "CLOSED" in store.info()
assert not store.is_open
store2.close()
assert "CLOSED" in store2.info()
assert not store2.is_open
# ops on a closed store
with ensure_clean_path(setup_path) as path:
df = tm.makeDataFrame()
df.to_hdf(path, "df", mode="w", format="table")
store = HDFStore(path)
store.close()
msg = r"[\S]* file is not open!"
with pytest.raises(ClosedFileError, match=msg):
store.keys()
with pytest.raises(ClosedFileError, match=msg):
"df" in store
with pytest.raises(ClosedFileError, match=msg):
len(store)
with pytest.raises(ClosedFileError, match=msg):
store["df"]
with pytest.raises(ClosedFileError, match=msg):
store.select("df")
with pytest.raises(ClosedFileError, match=msg):
store.get("df")
with pytest.raises(ClosedFileError, match=msg):
store.append("df2", df)
with pytest.raises(ClosedFileError, match=msg):
store.put("df3", df)
with pytest.raises(ClosedFileError, match=msg):
store.get_storer("df2")
with pytest.raises(ClosedFileError, match=msg):
store.remove("df2")
with pytest.raises(ClosedFileError, match=msg):
store.select("df")
msg = "'HDFStore' object has no attribute 'df'"
with pytest.raises(AttributeError, match=msg):
store.df
class HDFStoreDataFrame(BaseIO):
goal_time = 0.2
def setup(self):
N = 25000
index = tm.makeStringIndex(N)
self.df = DataFrame(
{
'float1': np.random.randn(N),
'float2': np.random.randn(N)
},
index=index)
self.df_mixed = DataFrame(
{
'float1': np.random.randn(N),
'float2': np.random.randn(N),
'string1': ['foo'] * N,
'bool1': [True] * N,
'int1': np.random.randint(0, N, size=N)
},
index=index)
self.df_wide = DataFrame(np.random.randn(N, 100))
self.start_wide = self.df_wide.index[10000]
self.stop_wide = self.df_wide.index[15000]
self.df2 = DataFrame(
{
'float1': np.random.randn(N),
'float2': np.random.randn(N)
},
index=date_range('1/1/2000', periods=N))
self.start = self.df2.index[10000]
self.stop = self.df2.index[15000]
self.df_wide2 = DataFrame(np.random.randn(N, 100),
index=date_range('1/1/2000', periods=N))
self.df_dc = DataFrame(np.random.randn(N, 10),
columns=['C%03d' % i for i in range(10)])
self.fname = '__test__.h5'
self.store = HDFStore(self.fname)
self.store.put('fixed', self.df)
self.store.put('fixed_mixed', self.df_mixed)
self.store.append('table', self.df2)
self.store.append('table_mixed', self.df_mixed)
self.store.append('table_wide', self.df_wide)
self.store.append('table_wide2', self.df_wide2)
def teardown(self):
self.store.close()
self.remove(self.fname)
def time_read_store(self):
self.store.get('fixed')
def time_read_store_mixed(self):
self.store.get('fixed_mixed')
def time_write_store(self):
self.store.put('fixed_write', self.df)
def time_write_store_mixed(self):
self.store.put('fixed_mixed_write', self.df_mixed)
def time_read_store_table_mixed(self):
self.store.select('table_mixed')
def time_write_store_table_mixed(self):
self.store.append('table_mixed_write', self.df_mixed)
def time_read_store_table(self):
self.store.select('table')
def time_write_store_table(self):
self.store.append('table_write', self.df)
def time_read_store_table_wide(self):
self.store.select('table_wide')
def time_write_store_table_wide(self):
self.store.append('table_wide_write', self.df_wide)
def time_write_store_table_dc(self):
self.store.append('table_dc_write', self.df_dc, data_columns=True)
def time_query_store_table_wide(self):
self.store.select('table_wide',
where="index > self.start_wide and "
"index < self.stop_wide")
def time_query_store_table(self):
self.store.select('table',
where="index > self.start and "
"index < self.stop")
def time_store_repr(self):
repr(self.store)
def time_store_str(self):
str(self.store)
def time_store_info(self):
self.store.info()
class HDFStoreDataFrame(BaseIO):
def setup(self):
N = 25000
index = tm.makeStringIndex(N)
self.df = DataFrame(
{"float1": np.random.randn(N), "float2": np.random.randn(N)}, index=index
)
self.df_mixed = DataFrame(
{
"float1": np.random.randn(N),
"float2": np.random.randn(N),
"string1": ["foo"] * N,
"bool1": [True] * N,
"int1": np.random.randint(0, N, size=N),
},
index=index,
)
self.df_wide = DataFrame(np.random.randn(N, 100))
self.start_wide = self.df_wide.index[10000]
self.stop_wide = self.df_wide.index[15000]
self.df2 = DataFrame(
{"float1": np.random.randn(N), "float2": np.random.randn(N)},
index=date_range("1/1/2000", periods=N),
)
self.start = self.df2.index[10000]
self.stop = self.df2.index[15000]
self.df_wide2 = DataFrame(
np.random.randn(N, 100), index=date_range("1/1/2000", periods=N)
)
self.df_dc = DataFrame(
np.random.randn(N, 10), columns=["C%03d" % i for i in range(10)]
)
self.fname = "__test__.h5"
self.store = HDFStore(self.fname)
self.store.put("fixed", self.df)
self.store.put("fixed_mixed", self.df_mixed)
self.store.append("table", self.df2)
self.store.append("table_mixed", self.df_mixed)
self.store.append("table_wide", self.df_wide)
self.store.append("table_wide2", self.df_wide2)
def teardown(self):
self.store.close()
self.remove(self.fname)
def time_read_store(self):
self.store.get("fixed")
def time_read_store_mixed(self):
self.store.get("fixed_mixed")
def time_write_store(self):
self.store.put("fixed_write", self.df)
def time_write_store_mixed(self):
self.store.put("fixed_mixed_write", self.df_mixed)
def time_read_store_table_mixed(self):
self.store.select("table_mixed")
def time_write_store_table_mixed(self):
self.store.append("table_mixed_write", self.df_mixed)
def time_read_store_table(self):
self.store.select("table")
def time_write_store_table(self):
self.store.append("table_write", self.df)
def time_read_store_table_wide(self):
self.store.select("table_wide")
def time_write_store_table_wide(self):
self.store.append("table_wide_write", self.df_wide)
def time_write_store_table_dc(self):
self.store.append("table_dc_write", self.df_dc, data_columns=True)
def time_query_store_table_wide(self):
self.store.select(
"table_wide", where="index > self.start_wide and index < self.stop_wide"
)
def time_query_store_table(self):
self.store.select("table", where="index > self.start and index < self.stop")
def time_store_repr(self):
repr(self.store)
def time_store_str(self):
str(self.store)
def time_store_info(self):
self.store.info()