def _returnIfTrue(self, data):
condition = self.filter_fn(data, *self.args, **self.kwargs)
if df_wrapper.is_column(condition):
# We are operating on DataFrames filter here
result = []
for idx, mask in enumerate(df_wrapper.iterate(condition)):
if mask:
result.append(df_wrapper.get_item(data, idx))
if len(result):
return df_wrapper.concat(result)
else:
return None
if not isinstance(condition, bool):
raise ValueError("Boolean output is required for `filter_fn` of FilterIterDataPipe, got", type(condition))
if condition:
return data
def __iter__(self):
for df in self.items:
for record in df_wrapper.iterate(df):
yield record
def __iter__(self):
for df in self.source_datapipe:
# for record in df.to_records(index=False):
for record in df_wrapper.iterate(df):
yield record
