def test_to_local_bounded_df(): df = ArrayDataFrame([[0, 1]], "a:int,b:int") idf = IterableDataFrame([[0, 1]], "a:int,b:int", dict(a=1)) assert to_local_bounded_df(df) is df r = to_local_bounded_df(idf) assert r is not idf assert r.as_array() == [[0, 1]] assert r.schema == "a:int,b:int" assert r.metadata == dict(a=1)
def run(self, cursor: PartitionCursor, dfs: DataFrames) -> LocalDataFrame: self.transformer._cursor = cursor # type: ignore if len(self.ignore_errors) == 0: return self.transformer.transform(dfs) else: try: return to_local_bounded_df(self.transformer.transform(dfs)) except self.ignore_errors: # type: ignore # pylint: disable=E0712 return ArrayDataFrame([], self.transformer.output_schema)
def run(self, cursor: PartitionCursor, df: LocalDataFrame) -> LocalDataFrame: self.transformer._cursor = cursor # type: ignore df._metadata = self.metadata if len(self.ignore_errors) == 0: return self.transformer.transform(df) else: try: return to_local_bounded_df(self.transformer.transform(df)) except self.ignore_errors: # type: ignore return ArrayDataFrame([], self.transformer.output_schema)
def to_df(self, df: Any, schema: Any = None, metadata: Any = None) -> LocalBoundedDataFrame: return to_local_bounded_df(df, schema, metadata)