def testToFrameOrSeries(self): raw = pd.Series(np.random.rand(10), name='col') series = Series(raw) r = series.to_frame() result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_frame_equal(raw.to_frame(), result) r = series.to_frame(name='new_name') result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_frame_equal(raw.to_frame(name='new_name'), result) raw = pd.Index(np.random.rand(10), name='col') index = Index(raw) r = index.to_frame() result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_frame_equal(raw.to_frame(), result) r = index.to_frame(index=False) result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_frame_equal(raw.to_frame(index=False), result) r = index.to_frame(name='new_name') result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_frame_equal(raw.to_frame(name='new_name'), result) r = index.to_series() result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_series_equal(raw.to_series(), result) r = index.to_series(index=pd.RangeIndex(0, 10)) result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_series_equal( raw.to_series(index=pd.RangeIndex(0, 10)), result) r = index.to_series(name='new_name') result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_series_equal(raw.to_series(name='new_name'), result) raw = pd.MultiIndex.from_tuples([('A', 'E'), ('B', 'F'), ('C', 'G')]) index = Index(raw, tupleize_cols=True) r = index.to_frame() result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_frame_equal(raw.to_frame(), result) with self.assertRaises(TypeError): index.to_frame(name='XY') with self.assertRaises(ValueError): index.to_frame(name=['X', 'Y', 'Z']) r = index.to_frame(name=['X', 'Y']) result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_frame_equal(raw.to_frame(name=['X', 'Y']), result) r = index.to_series(name='new_name') result = self.executor.execute_dataframe(r, concat=True)[0] pd.testing.assert_series_equal(raw.to_series(name='new_name'), result)
def test_series_params(): raw = pd.Series([1, 2, 3], name='a') series = Series(raw) series = series[series < 2] series = tile(series) c = series.chunks[0] assert any(np.isnan(s) for s in c.params['shape']) assert np.isnan(c.params['index_value'].min_val) c.params = c.get_params_from_data(raw[raw < 2]) # shape and index_value updated assert not any(np.isnan(s) for s in c.params['shape']) assert not np.isnan(c.params['index_value'].min_val) params = c.params.copy() params.pop('index', None) series.params = params assert np.prod(series.shape) > 0 series.refresh_params()
def test_groupby_params(): raw = pd.DataFrame({'a': [1, 2, 3]}) df = DataFrame(raw) grouped = df.groupby('a') grouped = tile(grouped) c = grouped.chunks[0] c.params = c.get_params_from_data(wrapped_groupby(raw, by='a')) params = c.params.copy() params.pop('index', None) grouped.params = params raw = pd.Series([1, 2, 3], name='a') series = Series(raw) grouped = series.groupby(level=0) grouped = tile(grouped) c = grouped.chunks[0] c.params = c.get_params_from_data(wrapped_groupby(raw, level=0)) params = c.params.copy() params.pop('index', None) grouped.params = params grouped.refresh_params()
def testKeyValue(self): raw = pd.DataFrame(np.random.rand(4, 3), columns=list('ABC')) df = DataFrame(raw) result = self.executor.execute_dataframe(df.values, concat=True)[0] np.testing.assert_array_equal(result, raw.values) result = self.executor.execute_dataframe(df.keys(), concat=True)[0] pd.testing.assert_index_equal(result, raw.keys()) raw = pd.Series(np.random.rand(10)) s = Series(raw) result = self.executor.execute_dataframe(s.values, concat=True)[0] np.testing.assert_array_equal(result, raw.values) result = self.executor.execute_dataframe(s.keys(), concat=True)[0] pd.testing.assert_index_equal(result, raw.keys()) raw = pd.Index(np.random.rand(10)) idx = Index(raw) result = self.executor.execute_dataframe(idx.values, concat=True)[0] np.testing.assert_array_equal(result, raw.values)
def test_key_value(setup): raw = pd.DataFrame(np.random.rand(4, 3), columns=list('ABC')) df = DataFrame(raw) result = df.values.execute().fetch() np.testing.assert_array_equal(result, raw.values) result = df.keys().execute().fetch() pd.testing.assert_index_equal(result, raw.keys()) raw = pd.Series(np.random.rand(10)) s = Series(raw) result = s.values.execute().fetch() np.testing.assert_array_equal(result, raw.values) result = s.keys().execute().fetch() pd.testing.assert_index_equal(result, raw.keys()) raw = pd.Index(np.random.rand(10)) idx = Index(raw) result = idx.values.execute().fetch() np.testing.assert_array_equal(result, raw.values)
def test_comp(setup): df1 = DataFrame(pd.DataFrame(np.random.rand(4, 3))) df2 = DataFrame(pd.DataFrame(np.random.rand(4, 3))) with enter_mode(build=True): assert not df1.data == df2.data assert df1.data == df1.data for op in [ operator.eq, operator.ne, operator.lt, operator.gt, operator.le, operator.ge ]: eq_df = op(df1, df2) pd.testing.assert_index_equal(eq_df.index_value.to_pandas(), df1.index_value.to_pandas()) # index not identical df3 = DataFrame(pd.DataFrame(np.random.rand(4, 3), index=[1, 2, 3, 4])) with pytest.raises(ValueError): op(df1, df3) # columns not identical df4 = DataFrame( pd.DataFrame(np.random.rand(4, 3), columns=['a', 'b', 'c'])) with pytest.raises(ValueError): op(df1, df4) # test datetime df = DataFrame(pd.DataFrame(pd.date_range('20130101', periods=6))) for op in [ operator.eq, operator.ne, operator.lt, operator.gt, operator.le, operator.ge ]: r_df = op(df, datetime(2013, 1, 2)) pd.testing.assert_index_equal(r_df.index_value.to_pandas(), df.index_value.to_pandas()) # test period type raw = pd.period_range("2000-01-01", periods=10, freq="D") raw_series = pd.Series(raw) series = Series(raw, chunk_size=5) r = series >= series[1] pd.testing.assert_series_equal(r.to_pandas(), raw_series >= raw_series[1])
def test_index_params(): raw = pd.Series([1, 2, 3], name='a') raw.index.name = 'b' series = Series(raw) series = series[series < 2] index = series.index index = tile(index) c = index.chunks[0] assert any(np.isnan(s) for s in c.params['shape']) assert np.isnan(c.params['index_value'].min_val) c.params = c.get_params_from_data(raw[raw < 2].index) # shape and index_value updated assert not any(np.isnan(s) for s in c.params['shape']) assert not np.isnan(c.params['index_value'].min_val) params = c.params.copy() params.pop('index', None) index.params = params assert np.prod(index.shape) > 0 index.refresh_params()
def test_to_frame_or_series(setup): raw = pd.Series(np.random.rand(10), name='col') series = Series(raw) r = series.to_frame() result = r.execute().fetch() pd.testing.assert_frame_equal(raw.to_frame(), result) r = series.to_frame(name='new_name') result = r.execute().fetch() pd.testing.assert_frame_equal(raw.to_frame(name='new_name'), result) series = series[series > 0.1] r = series.to_frame(name='new_name') result = r.execute().fetch() pd.testing.assert_frame_equal(raw[raw > 0.1].to_frame(name='new_name'), result) raw = pd.Index(np.random.rand(10), name='col') index = Index(raw) r = index.to_frame() result = r.execute().fetch() pd.testing.assert_frame_equal(raw.to_frame(), result) r = index.to_frame(index=False) result = r.execute().fetch() pd.testing.assert_frame_equal(raw.to_frame(index=False), result) r = index.to_frame(name='new_name') result = r.execute().fetch() pd.testing.assert_frame_equal(raw.to_frame(name='new_name'), result) r = index.to_series() result = r.execute().fetch() pd.testing.assert_series_equal(raw.to_series(), result) r = index.to_series(index=pd.RangeIndex(0, 10)) result = r.execute().fetch() pd.testing.assert_series_equal(raw.to_series(index=pd.RangeIndex(0, 10)), result) r = index.to_series(name='new_name') result = r.execute().fetch() pd.testing.assert_series_equal(raw.to_series(name='new_name'), result) raw = pd.MultiIndex.from_tuples([('A', 'E'), ('B', 'F'), ('C', 'G')]) index = Index(raw, tupleize_cols=True) r = index.to_frame() result = r.execute().fetch() pd.testing.assert_frame_equal(raw.to_frame(), result) with pytest.raises(TypeError): index.to_frame(name='XY') with pytest.raises(ValueError): index.to_frame(name=['X', 'Y', 'Z']) r = index.to_frame(name=['X', 'Y']) result = r.execute().fetch() pd.testing.assert_frame_equal(raw.to_frame(name=['X', 'Y']), result) r = index.to_series(name='new_name') result = r.execute().fetch() pd.testing.assert_series_equal(raw.to_series(name='new_name'), result)