def _map(pdf: pd.DataFrame) -> pd.DataFrame:
if len(presort_keys) > 0:
pdf = pdf.sort_values(presort_keys, ascending=presort_asc)
input_df = PandasDataFrame(pdf.reset_index(drop=True),
df.schema,
pandas_df_wrapper=True)
cursor.set(input_df.peek_array(), cursor.partition_no + 1, 0)
output_df = map_func(cursor, input_df)
return output_df.as_pandas()
def test_simple_methods():
df = PandasDataFrame([], "a:str,b:int")
assert df.as_pandas() is df.native
assert df.empty
assert 0 == df.count()
assert df.is_local
df = PandasDataFrame([["a", 1], ["b", "2"]], "x:str,y:double")
assert df.as_pandas() is df.native
assert not df.empty
assert 2 == df.count()
assert ["a", 1.0] == df.peek_array()
assert dict(x="a", y=1.0) == df.peek_dict()
def _map(pdf: Any) -> pd.DataFrame:
if pdf.shape[0] == 0:
return PandasDataFrame([], output_schema).as_pandas()
if len(presort_keys) > 0:
pdf = pdf.sort_values(presort_keys, ascending=presort_asc)
input_df = PandasDataFrame(pdf.reset_index(drop=True),
input_schema,
pandas_df_wrapper=True)
if on_init_once is not None:
on_init_once(0, input_df)
cursor = partition_spec.get_cursor(input_schema, 0)
cursor.set(input_df.peek_array(), 0, 0)
output_df = map_func(cursor, input_df)
return output_df.as_pandas()