def generate_statistics_from_dataframe( dataframe: DataFrame, stats_options: options.StatsOptions = options.StatsOptions(), n_jobs: int = 1) -> statistics_pb2.DatasetFeatureStatisticsList: """Compute data statistics for the input pandas DataFrame. This is a utility method for users with in-memory data represented as a pandas DataFrame. Args: dataframe: Input pandas DataFrame. stats_options: `tfdv.StatsOptions` for generating data statistics. n_jobs: Number of processes to run (defaults to 1). If -1 is provided, uses the same number of processes as the number of CPU cores. Returns: A DatasetFeatureStatisticsList proto. """ if not isinstance(dataframe, DataFrame): raise TypeError('dataframe argument is of type {}. Must be a ' 'pandas DataFrame.'.format(type(dataframe).__name__)) stats_generators = cast( List[stats_generator.CombinerStatsGenerator], stats_impl.get_generators(stats_options, in_memory=True)) if n_jobs < -1 or n_jobs == 0: raise ValueError('Invalid n_jobs parameter {}. Should be either ' ' -1 or >= 1.'.format(n_jobs)) if n_jobs == -1: n_jobs = multiprocessing.cpu_count() n_jobs = max(min(n_jobs, multiprocessing.cpu_count()), 1) if n_jobs == 1: merged_partial_stats = _generate_partial_statistics_from_df( dataframe, stats_options, stats_generators) else: # TODO(pachristopher): Investigate why we don't observe linear speedup after # a certain number of processes. splits = np.array_split(dataframe, n_jobs) partial_stats = Parallel(n_jobs=n_jobs)( delayed(_generate_partial_statistics_from_df)( splits[i], stats_options, stats_generators) for i in range(n_jobs)) merged_partial_stats = [ gen.merge_accumulators(stats) for gen, stats in zip(stats_generators, zip(*partial_stats)) ] return stats_impl.extract_statistics_output(merged_partial_stats, stats_generators)
def generate_statistics_from_dataframe( dataframe, stats_options = options.StatsOptions(), n_jobs = multiprocessing.cpu_count() ): """Compute data statistics for the input pandas DataFrame. This is a utility method for users with in-memory data represented as a pandas DataFrame. Args: dataframe: Input pandas DataFrame. stats_options: Options for generating data statistics. n_jobs: Number of processes to run. Returns: A DatasetFeatureStatisticsList proto. """ if not isinstance(dataframe, pd.DataFrame): raise TypeError('dataframe argument is of type {}. Must be a ' 'pandas DataFrame.'.format(type(dataframe).__name__)) stats_generators = stats_impl.get_generators(stats_options, in_memory=True) n_jobs = max(min(n_jobs, multiprocessing.cpu_count()), 1) if n_jobs == 1: merged_partial_stats = _generate_partial_statistics_from_df( dataframe, stats_options, stats_generators) else: # TODO(pachristopher): Investigate why we don't observe linear speedup after # a certain number of processes. splits = np.array_split(dataframe, n_jobs) partial_stats = Parallel(n_jobs=n_jobs)( delayed(_generate_partial_statistics_from_df)( splits[i], stats_options, stats_generators) for i in range(n_jobs)) merged_partial_stats = [ gen.merge_accumulators(stats) for gen, stats in zip(stats_generators, zip(*partial_stats)) ] return stats_impl.extract_statistics_output( merged_partial_stats, stats_generators)