def print_data(df: SparkDataFrame, row_count: int = 10) -> PandasDataFrame: """Print the data count and data content of given spark data frame. :param df: the spark data frame to show the data :param row_count: the number of data to print :return: the truncated and transformed data frame with row_count """ print(f'data count: {df.count()}') return df.limit(row_count).toPandas()
def save(df: DataFrame, logger: Logger): logger.info(f"Saving {df.count()} records") number_of_mvu = 10 return df.limit(number_of_mvu)
def displayLocal(df: DataFrame): print(df.limit(100).toPandas())