def test_append_table(self):
with TemporaryDirectory() as td:
store = tb.OBTFile(td+'/test.h5', 'w', 'symbol')
store['AAPL'] = df
# Pytable table
table = store.obt.table.obj
colnames = table.colnames
# index name, columns, frame_key
assert colnames == ['timestamp', 'open', 'high', 'low', 'close', 'vol', 'other_dates', 'symbol']
temp = store.ix['AAPL']
tm.assert_frame_equal(temp, df, "dataframe returned from HDF is different", check_names=False)
# this shoudl throw error
bad_df = df.reindex(columns=['high', 'low', 'open', 'close', 'other_dates', 'vol'])
try:
store['BAD'] = bad_df
except tb.MismatchColumnsError as e:
pass # success
else:
assert False, 'this should return an error, columns in wrong order'
# fails when obt doesn't check column order. stored wrong
temp = store.ix['BAD']
if len(temp):
# if properly throwing erros, we should never reach here
tm.assert_frame_equal(temp, df, "dataframe returned from HDF is different")
def test_downsample_drop_empty(self):
"""
the drop_empty which is the default will not include
empty groups into the GroupBy.
"""
grouped = df.downsample('D')
test = grouped.mean()
correct = df.resample('D', how='mean').dropna(how='all')
tm.assert_frame_equal(test, correct)
def test_downsample_drop_empty(self):
"""
the drop_empty which is the default will not include
empty groups into the GroupBy.
"""
grouped = df.downsample('D')
test = grouped.mean()
correct = df.resample('D', how='mean').dropna(how='all')
tm.assert_frame_equal(test, correct)
def assert_op(pobj, userobj, op):
return
cls = type(userobj)
correct = op(pobj)
test = op(userobj)
assert isinstance(test, cls)
if isinstance(correct, pd.Series):
tm.assert_series_equal(correct, test)
if isinstance(correct, pd.DataFrame):
tm.assert_frame_equal(correct, test)
def assert_op(pobj, userobj, op):
return
cls = type(userobj)
correct = op(pobj)
test = op(userobj)
assert isinstance(test, cls)
if isinstance(correct, pd.Series):
tm.assert_series_equal(correct, test)
if isinstance(correct, pd.DataFrame):
tm.assert_frame_equal(correct, test)
def test_pairwise(self):
df = pd.DataFrame(index=list(range(10)))
for x in range(3):
df[x] = list(range(x, x + 10))
nandf = df.copy().astype(float)
nandf.ix[9:, 1] = np.nan
# test with order=True
# test with permutations
pairs = pairwise(df, lambda x, y: x.sum() - y.sum())
expected = pd.DataFrame([[0, -10, -20], [10, 0, -10], [20, 10, 0]],
index=list(range(3)),
dtype=float)
tm.assert_frame_equal(pairs, expected)
# test with combinations
pairs = pairwise(df, lambda x, y: x.sum() - y.sum(), order=False)
expected = pd.DataFrame([[0, -10, -20], [-10, 0, -10], [-20, -10, 0]],
index=list(range(3)),
dtype=float)
tm.assert_frame_equal(pairs, expected)
# test with combinations and values
# use nandf to test. np.ndarray.sum() returns NaN if it contains nan
pairs = pairwise(nandf,
lambda x, y: x.sum() - y.sum(),
order=True,
force_values=True)
expected = pd.DataFrame(
[[0, np.nan, -20], [np.nan, np.nan, np.nan], [20, np.nan, 0]],
index=list(range(3)),
dtype=float)
tm.assert_frame_equal(pairs, expected)
# test with np.nansum.
pairs = pairwise(nandf,
lambda x, y: np.nansum(x) - np.nansum(y),
order=True,
force_values=True)
expected = pd.DataFrame([[0, 0, -20], [0, 0, -20], [20, 20, 0]],
index=list(range(3)),
dtype=float)
tm.assert_frame_equal(pairs, expected)
# the np.nansum version should be same as Series.sum version
pairs_series = pairwise(nandf,
lambda x, y: x.sum() - y.sum(),
order=True,
force_values=False)
tm.assert_frame_equal(pairs, pairs_series)
def test_to_frame_override(self):
"""
Test to_frame(attrs, repr_col)
"""
ind = pd.date_range(start="2000/1/1", periods=10)
events = [TestObject(ind[i], i, len(ind)-i) for i in range(len(ind))]
el = lf.ListFrame(events, attrs=['id'], repr_col=True)
correct = el.to_frame()
el2 = lf.ListFrame(events)
bad = el2.to_frame() # regular ole to_frame
assert set(bad.columns) == set(TestObject._frame_cols)
test = el2.to_frame(attrs=['id'], repr_col=True)
tm.assert_frame_equal(test, correct)
def test_pairwise(self):
df = pd.DataFrame(index=range(10))
for x in range(3):
df[x] = range(x, x+10)
nandf = df.copy().astype(float)
nandf.ix[9:,1] = np.nan
# test with order=True
# test with permutations
pairs = pairwise(df, lambda x, y: x.sum() - y.sum())
expected = pd.DataFrame([[0, -10, -20],
[10, 0, -10],
[20, 10, 0]], index=range(3), dtype=float)
tm.assert_frame_equal(pairs, expected)
# test with combinations
pairs = pairwise(df, lambda x, y: x.sum() - y.sum(), order=False)
expected = pd.DataFrame([[0, -10, -20],
[-10, 0, -10],
[-20, -10, 0]], index=range(3), dtype=float)
tm.assert_frame_equal(pairs, expected)
# test with combinations and values
# use nandf to test. np.ndarray.sum() returns NaN if it contains nan
pairs = pairwise(nandf, lambda x, y: x.sum() - y.sum(), order=True,
force_values=True)
expected = pd.DataFrame([[0, np.nan, -20],
[np.nan, np.nan, np.nan],
[20, np.nan, 0]], index=range(3), dtype=float)
tm.assert_frame_equal(pairs, expected)
# test with np.nansum.
pairs = pairwise(nandf, lambda x, y: np.nansum(x) - np.nansum(y),
order=True, force_values=True)
expected = pd.DataFrame([[0, 0, -20],
[0, 0, -20],
[20, 20, 0]], index=range(3), dtype=float)
tm.assert_frame_equal(pairs, expected)
# the np.nansum version should be same as Series.sum version
pairs_series = pairwise(nandf, lambda x, y: x.sum() - y.sum(),
order=True, force_values=False)
tm.assert_frame_equal(pairs, pairs_series)
def test_append_table(self):
with TemporaryDirectory() as td:
store = tb.OBTFile(td + '/test.h5', 'w', 'symbol')
store['AAPL'] = df
# Pytable table
table = store.obt.table.obj
colnames = table.colnames
# index name, columns, frame_key
assert colnames == [
'timestamp', 'open', 'high', 'low', 'close', 'vol',
'other_dates', 'symbol'
]
temp = store.ix['AAPL']
tm.assert_frame_equal(temp,
df,
"dataframe returned from HDF is different",
check_names=False)
# this shoudl throw error
bad_df = df.reindex(
columns=['high', 'low', 'open', 'close', 'other_dates', 'vol'])
try:
store['BAD'] = bad_df
except tb.MismatchColumnsError as e:
pass # success
else:
assert False, 'this should return an error, columns in wrong order'
# fails when obt doesn't check column order. stored wrong
temp = store.ix['BAD']
if len(temp):
# if properly throwing erros, we should never reach here
tm.assert_frame_equal(
temp, df, "dataframe returned from HDF is different")
import trtools.util.testing as tm
import trtools.io.api as trio
import trtools.tools.pload as pload
# TODO add a generic testing file
import sys; sys.exit(0)
store = trio.MetaFileCache('/Volumes/tradedata/dataload/mb_data2', leveled=2)
d = pload.pload(store)
for k in d:
test = d[k]
correct = store[k]
tm.assert_frame_equal(test, correct)
def test_select_translate_panel(self):
"""
Test the dictionary translate
"""
df1 = tm.makeTimeDataFrame()
df2 = tm.makeTimeDataFrame()
df3 = tm.makeTimeDataFrame()
panel = pd.Panel({111: df1, 123: df2, 666:df3})
# setup the translation
sl.KEY_TRANS = {'dale': 111, 'bob': 123}
test = panel.ix["dale"]
tm.assert_frame_equal(test, df1)
tm.assert_frame_equal(panel.ix[123], df2)
tm.assert_frame_equal(panel.ix['bob'], df2)
tm.assert_frame_equal(panel.ix['bob', :10], df2.ix[:10])
tm.assert_frame_equal(panel.ix['bob', :10, :3], df2.ix[:10, :3])
# grab sub panel
test = panel.ix[["dale", "bob"]]
assert np.all(test.items == [111, 123])
tm.assert_frame_equal(test.ix['dale'], df1)
tm.assert_frame_equal(test.ix['bob'], df2)
sl.KEY_TRANS = None
def downsample(self):
# these should be equivalent
grouped = df.downsample('D', drop_empty=False)
test = grouped.mean()
correct = df.resample('D', how='mean')
tm.assert_frame_equal(test, correct)
def test_downsample(self):
# these should be equivalent
grouped = df.downsample('D', drop_empty=False)
test = grouped.mean()
correct = df.resample('D', how='mean')
tm.assert_frame_equal(test, correct)
def test_select_translate_panel(self):
"""
Test the dictionary translate
"""
df1 = tm.makeTimeDataFrame()
df2 = tm.makeTimeDataFrame()
df3 = tm.makeTimeDataFrame()
panel = pd.Panel({111: df1, 123: df2, 666: df3})
# setup the translation
sl.KEY_TRANS = {'dale': 111, 'bob': 123}
test = panel.ix["dale"]
tm.assert_frame_equal(test, df1)
tm.assert_frame_equal(panel.ix[123], df2)
tm.assert_frame_equal(panel.ix['bob'], df2)
tm.assert_frame_equal(panel.ix['bob', :10], df2.ix[:10])
tm.assert_frame_equal(panel.ix['bob', :10, :3], df2.ix[:10, :3])
# grab sub panel
test = panel.ix[["dale", "bob"]]
assert np.all(test.items == [111, 123])
tm.assert_frame_equal(test.ix['dale'], df1)
tm.assert_frame_equal(test.ix['bob'], df2)
sl.KEY_TRANS = None