def _simple_entropy(df: pyspark.sql.dataframe.DataFrame, column_name: str) -> float: count = df.count() testdf = df.select(column_name).groupby(column_name).agg((F.count(column_name) / count).alias("p")) result = testdf.groupby().agg(-F.sum(F.col("p") * F.log2("p"))).collect()[0][0] if not result: return 0.0 return result
def _find_best_split( countdf: pyspark.sql.dataframe.DataFrame, prev_split_columns: List[str], valid_column_names: List[str], target_column_name: str, normalization: Optional[Dict[str, int]] = None, ) -> Tuple[float, str]: total_count = countdf.count() max_score_tuple = 0.0, None pre_split_entropy = _weighted_entropy(countdf, total_count, prev_split_columns, target_column_name, True) for column_name in valid_column_names: if column_name == target_column_name: continue new_split_columns = prev_split_columns[:] new_split_columns.append(column_name) post_split_entropy = _weighted_entropy(countdf, total_count, new_split_columns, target_column_name, True) value = pre_split_entropy - post_split_entropy if normalization and normalization[column_name] > 0: value /= math.log(normalization[column_name]) if value > max_score_tuple[0]: max_score_tuple = value, column_name return max_score_tuple
def get_repartition_value(sdf: pyspark.sql.dataframe.DataFrame, target_size: int = 245, compression: str = 'none') -> int: lenght = sdf.count() df_1_row = sdf.limit(int(1e4)) tmp_file_name = 'test_file' while check_hdfs_file_ex(tmp_file_name): tmp_file_name += '_' df_1_row.coalesce(1).write.option('compression', compression)\ .mode('overwrite').parquet(tmp_file_name) row_byte_weight = int(sh.hdfs('dfs', '-du', tmp_file_name)\ .stdout.decode('utf-8').split('\n')[-2].split(' ')[0]) sh.hdfs('dfs', '-rm', '-R', '-skipTrash', tmp_file_name) nd_rep_val = int(row_byte_weight * lenght / target_size / (1024 * 1024) / 1e4) return 1 if nd_rep_val < 1 else nd_rep_val
def estimate_segments( df: pyspark.sql.dataframe.DataFrame, target_field: str = None, max_segments: int = 30, include_columns: List[str] = [], unique_perc_bounds: Tuple[float, float] = [None, 0.8], null_perc_bounds: Tuple[float, float] = [None, 0.2], ) -> Optional[Union[List[Dict], List[str]]]: """ Estimates the most important features and values on which to segment data profiling using entropy-based methods. If no target column provided, maximum entropy column is substituted. :param df: the dataframe of data to profile :param target_field: target field (optional) :param max_segments: upper threshold for total combinations of segments, default 30 :param include_columns: additional non-string columns to consider in automatic segmentation. Warning: high cardinality columns will degrade performance. :param unique_perc_bounds: tuple of form [lower, upper] with bounds on the percentage of unique values (|unique| / |X|). Upper bound exclusive. :param null_perc_bounds: tuple of form [lower, upper] with bounds on the percentage of null values. Upper bound exclusive. :return: a list of segmentation feature names """ current_split_columns = [] segments = [] segments_used = 1 max_entropy_column = (float("-inf"), None) if not unique_perc_bounds[0]: unique_perc_bounds[0] = float("-inf") if not unique_perc_bounds[1]: unique_perc_bounds[1] = float("inf") if not null_perc_bounds[0]: null_perc_bounds[0] = float("-inf") if not null_perc_bounds[1]: null_perc_bounds[1] = float("inf") valid_column_names = set() count = df.count() print("Limiting to categorical (string) data columns...") valid_column_names = {col for col in df.columns if (df.select(col).dtypes[0][1] == "string" or col in include_columns)} print("Gathering cardinality information...") n_uniques = {col: df.agg(F.approx_count_distinct(col)).collect()[0][0] for col in valid_column_names} print("Gathering missing value information...") n_nulls = {col: df.filter(df[col].isNull()).count() for col in valid_column_names} print("Finding valid columns for autosegmentation...") for col in valid_column_names.copy(): null_perc = 0.0 if count == 0 else n_nulls[col] / count unique_perc = 0.0 if count == 0 else n_uniques[col] / count if ( col in segments or n_uniques[col] <= 1 or null_perc < null_perc_bounds[0] or null_perc >= null_perc_bounds[1] or unique_perc < unique_perc_bounds[0] or unique_perc >= unique_perc_bounds[1] ): valid_column_names.remove(col) if not valid_column_names: return [] if not target_field: print("Finding alternative target field since none were specified...") for col in valid_column_names: col_entropy = _simple_entropy(df, col) if n_uniques[col] > 1: col_entropy /= math.log(n_uniques[col]) if col_entropy > max_entropy_column[0]: max_entropy_column = (col_entropy, col) target_field = max_entropy_column[1] print(f"Using {target_field} column as target field.") assert target_field in df.columns valid_column_names.add(target_field) valid_column_names = list(valid_column_names) countdf = df.select(valid_column_names).groupby(valid_column_names).count().cache() print("Calculating segments...") while segments_used < max_segments: valid_column_names = {col for col in valid_column_names if (col not in segments and n_uniques[col] * segments_used <= (max_segments - segments_used))} _, segment_column_name = _find_best_split( countdf, current_split_columns, list(valid_column_names), target_column_name=target_field, normalization=n_uniques ) if not segment_column_name: break segments.append(segment_column_name) current_split_columns.append(segment_column_name) segments_used *= n_uniques[segment_column_name] return segments
def test_calculate_material_change( merge_data: pyspark.sql.dataframe.DataFrame, last_week_sale: pyspark.sql.dataframe.DataFrame): assert merge_data.count() == last_week_sale.count(), 'we want ' + str(last_week_sale.count()) + \ ". But we get "+str(merge_data.count())# test joined dataset