def get_messages(table_stats, series_description, correlations): messages = check_table_messages(table_stats) for col, description in series_description.items(): messages += check_variable_messages(col, description) messages += check_correlation_messages(correlations) messages.sort(key=lambda message: str(message.message_type)) return messages
def get_messages(config: Settings, table_stats: dict, series_description: dict, correlations: dict) -> List[Message]: messages = check_table_messages(table_stats) for col, description in series_description.items(): messages += check_variable_messages(config, col, description) messages += check_correlation_messages(config, correlations) messages.sort(key=lambda message: str(message.message_type)) return messages
def describe(df: pd.DataFrame) -> dict: """Calculate the statistics for each series in this DataFrame. Args: df: DataFrame. Returns: This function returns a dictionary containing: - table: overall statistics. - variables: descriptions per series. - correlations: correlation matrices. - missing: missing value diagrams. - messages: direct special attention to these patterns in your data. """ if not isinstance(df, pd.DataFrame): warnings.warn("df is not of type pandas.DataFrame") if df.empty: raise ValueError("df can not be empty") disable_progress_bar = not config["progress_bar"].get(bool) # Multiprocessing of Describe 1D for each column pool_size = config["pool_size"].get(int) if pool_size <= 0: pool_size = multiprocessing.cpu_count() args = [(column, series) for column, series in df.iteritems()] series_description = {} with tqdm(total=len(args), desc="variables", disable=disable_progress_bar) as pbar: if pool_size == 1: for arg in args: column, description = multiprocess_1d(arg) series_description[column] = description pbar.update() else: # Store the original order original_order = { k: v for v, k in enumerate([column for column, _ in args]) } # TODO: use `Pool` for Linux-based systems with multiprocessing.pool.ThreadPool(pool_size) as executor: for i, (column, description) in enumerate( executor.imap_unordered(multiprocess_1d, args) ): series_description[column] = description pbar.set_postfix({'feature_name': column}) pbar.update() # Restore the original order series_description = dict( sorted( series_description.items(), key=lambda index: original_order.get(index[0]), ) ) # Mapping from column name to variable type sort = config["sort"].get(str) series_description = sort_column_names(series_description, sort) variables = { column: description["type"] for column, description in series_description.items() } # Transform the series_description in a DataFrame variable_stats = pd.DataFrame(series_description) # Get correlations correlations = calculate_correlations(df, variables) # Scatter matrix scatter_matrix = get_scatter_matrix(df, variables) # Table statistics with tqdm(total=1, desc="table", disable=disable_progress_bar) as pbar: table_stats = describe_table(df, variable_stats) pbar.update(1) # missing diagrams missing = get_missing_diagrams(df, table_stats) # Messages with tqdm(total=3, desc="warnings", disable=disable_progress_bar) as pbar: pbar.set_description_str("warnings [table]") messages = check_table_messages(table_stats) pbar.update() pbar.set_description_str("warnings [variables]") for col, description in series_description.items(): messages += check_variable_messages(col, description) pbar.update() pbar.set_description_str("warnings [correlations]") messages += check_correlation_messages(correlations) messages.sort(key=lambda message: str(message.message_type)) pbar.update() with tqdm(total=1, desc="package", disable=disable_progress_bar) as pbar: package = { "pandas_profiling_version": __version__, "pandas_profiling_config": config.dump(), } pbar.update() return { # Overall description "table": table_stats, # Per variable descriptions "variables": series_description, # Bivariate relations "scatter": scatter_matrix, # Correlation matrices "correlations": correlations, # Missing values "missing": missing, # Warnings "messages": messages, # Package "package": package, }
def describe(df: pd.DataFrame) -> dict: """Calculate the statistics for each series in this DataFrame. Args: df: DataFrame. Returns: This function returns a dictionary containing: - table: overall statistics. - variables: descriptions per series. - correlations: correlation matrices. - missing: missing value diagrams. - messages: direct special attention to these patterns in your data. """ if not isinstance(df, pd.DataFrame): raise TypeError("df must be of type pandas.DataFrame") if df.empty: raise ValueError("df can not be empty") # Multiprocessing of Describe 1D for each column pool_size = config["pool_size"].get(int) if pool_size <= 0: pool_size = multiprocessing.cpu_count() if pool_size == 1: args = [(column, series) for column, series in df.iteritems()] series_description = { column: series for column, series in itertools.starmap(multiprocess_1d, args) } else: with multiprocessing.pool.ThreadPool(pool_size) as executor: series_description = {} results = executor.starmap(multiprocess_1d, df.iteritems()) for col, description in results: series_description[col] = description # Mapping from column name to variable type variables = { column: description["type"] for column, description in series_description.items() } # Get correlations correlations = calculate_correlations(df, variables) # Scatter matrix scatter_matrix = get_scatter_matrix(df, variables) # Transform the series_description in a DataFrame variable_stats = pd.DataFrame(series_description) # Table statistics table_stats = describe_table(df, variable_stats) # missing diagrams missing = get_missing_diagrams(df, table_stats) # Messages messages = check_table_messages(table_stats) for col, description in series_description.items(): messages += check_variable_messages(col, description) messages += check_correlation_messages(correlations) package = { "pandas_profiling_version": __version__, "pandas_profiling_config": config.dump(), } return { # Overall description "table": table_stats, # Per variable descriptions "variables": series_description, # Bivariate relations "scatter": scatter_matrix, # Correlation matrices "correlations": correlations, # Missing values "missing": missing, # Warnings "messages": messages, # Package "package": package, }