def _update_example_and_missing_count(stats):
"""Updates example count of the dataset and missing count for all features."""
dummy_feature = stats_util.get_feature_stats(stats, _DUMMY_FEATURE_NAME)
num_examples = stats_util.get_custom_stats(dummy_feature,
_NUM_EXAMPLES_KEY)
weighted_num_examples = stats_util.get_custom_stats(
dummy_feature, _WEIGHTED_NUM_EXAMPLES_KEY)
stats.features.remove(dummy_feature)
for feature_stats in stats.features:
common_stats = None
if feature_stats.WhichOneof('stats') == 'num_stats':
common_stats = feature_stats.num_stats.common_stats
else:
common_stats = feature_stats.string_stats.common_stats
assert num_examples >= common_stats.num_non_missing, (
'Total number of examples: {} is less than number of non missing '
'examples: {} for feature {}.'.format(num_examples,
common_stats.num_non_missing,
feature_stats.name))
common_stats.num_missing = int(num_examples -
common_stats.num_non_missing)
if weighted_num_examples != 0:
common_stats.weighted_common_stats.num_missing = (
weighted_num_examples -
common_stats.weighted_common_stats.num_non_missing)
stats.num_examples = int(num_examples)
def _update_example_and_missing_count(
stats: statistics_pb2.DatasetFeatureStatistics) -> None:
"""Updates example count of the dataset and missing count for all features."""
if not stats.features:
return
dummy_feature = stats_util.get_feature_stats(stats, _DUMMY_FEATURE_PATH)
num_examples = stats_util.get_custom_stats(dummy_feature,
_NUM_EXAMPLES_KEY)
weighted_num_examples = stats_util.get_custom_stats(
dummy_feature, _WEIGHTED_NUM_EXAMPLES_KEY)
stats.features.remove(dummy_feature)
for feature_stats in stats.features:
# For features nested under a STRUCT feature, their num_missing is computed
# in the basic stats generator (because their num_missing is relative to
# their parent's value count).
if len(feature_stats.path.step) > 1:
continue
common_stats = None
which_oneof_stats = feature_stats.WhichOneof('stats')
if which_oneof_stats is None:
# There are not common_stats for this feature (which can be the case when
# generating only custom_stats for a sparse or weighted feature). In that
# case, simply continue without modifying the common stats.
continue
common_stats = getattr(feature_stats, which_oneof_stats).common_stats
assert num_examples >= common_stats.num_non_missing, (
'Total number of examples: {} is less than number of non missing '
'examples: {} for feature {}.'.format(
num_examples, common_stats.num_non_missing,
'.'.join(feature_stats.path.step)))
num_missing = int(num_examples - common_stats.num_non_missing)
common_stats.num_missing = num_missing
if common_stats.presence_and_valency_stats:
common_stats.presence_and_valency_stats[
0].num_missing = num_missing
if weighted_num_examples != 0:
weighted_num_missing = (
weighted_num_examples -
common_stats.weighted_common_stats.num_non_missing)
common_stats.weighted_common_stats.num_missing = weighted_num_missing
if common_stats.weighted_presence_and_valency_stats:
common_stats.weighted_presence_and_valency_stats[
0].num_missing = (weighted_num_missing)
stats.num_examples = int(num_examples)
stats.weighted_num_examples = weighted_num_examples
def get_num_feature_stats_as_dataframe(
stats_list: statistics_pb2.DatasetFeatureStatisticsList,
feature_path: FeaturePath):
"""Returns a series of numeric statistics for a given
feature formatted as a tidy dataframe."""
feature_stats_list = []
for dataset in stats_list.datasets:
if dataset.name != 'All Examples':
feature_stats = get_feature_stats(dataset, feature_path)
if not feature_stats.HasField('num_stats'):
raise ValueError('This is not a numeric feature')
stats_dict = MessageToDict(feature_stats.num_stats)
del stats_dict['commonStats']
del stats_dict['histograms']
stats_dict['slice'] = dataset.name
feature_stats_list.append(stats_dict)
return pd.DataFrame(feature_stats_list)
def get_histograms_as_dataframe(
stats_list: statistics_pb2.DatasetFeatureStatisticsList,
feature_path: FeaturePath):
"""Returns a series of histograms for a given numeric feature
formatted as a tidy dataframe"""
buckets = []
for dataset in stats_list.datasets:
if dataset.name != 'All Examples':
feature_stats = get_feature_stats(dataset, feature_path)
if not feature_stats.HasField('num_stats'):
raise ValueError('This is not a numeric feature')
for histogram in feature_stats.num_stats.histograms:
if histogram.type == statistics_pb2.Histogram.HistogramType.STANDARD:
for bucket in histogram.buckets:
bucket_dict = MessageToDict(bucket)
bucket_dict['slice'] = dataset.name
buckets.append(bucket_dict)
return pd.DataFrame(buckets)
def _update_example_and_missing_count(stats):
"""Updates example count of the dataset and missing count for all features."""
dummy_feature = stats_util.get_feature_stats(stats, _DUMMY_FEATURE_NAME)
num_examples = stats_util.get_custom_stats(dummy_feature,
_NUM_EXAMPLES_KEY)
weighted_num_examples = stats_util.get_custom_stats(
dummy_feature, _WEIGHTED_NUM_EXAMPLES_KEY)
stats.features.remove(dummy_feature)
for feature_stats in stats.features:
common_stats = None
if feature_stats.WhichOneof('stats') == 'num_stats':
common_stats = feature_stats.num_stats.common_stats
else:
common_stats = feature_stats.string_stats.common_stats
common_stats.num_missing = int(num_examples -
common_stats.num_non_missing)
if weighted_num_examples != 0:
common_stats.weighted_common_stats.num_missing = (
weighted_num_examples -
common_stats.weighted_common_stats.num_non_missing)
stats.num_examples = int(num_examples)
def _update_example_and_missing_count(
stats: statistics_pb2.DatasetFeatureStatistics) -> None:
"""Updates example count of the dataset and missing count for all features."""
if not stats.features:
return
dummy_feature = stats_util.get_feature_stats(stats, _DUMMY_FEATURE_PATH)
num_examples = stats_util.get_custom_stats(dummy_feature,
_NUM_EXAMPLES_KEY)
weighted_num_examples = stats_util.get_custom_stats(
dummy_feature, _WEIGHTED_NUM_EXAMPLES_KEY)
stats.features.remove(dummy_feature)
for feature_stats in stats.features:
# For features nested under a STRUCT feature, their num_missing is computed
# in the basic stats generator (because their num_missing is relative to
# their parent's value count).
if len(feature_stats.path.step) > 1:
continue
common_stats = None
which_oneof_stats = feature_stats.WhichOneof('stats')
if which_oneof_stats == 'num_stats':
common_stats = feature_stats.num_stats.common_stats
elif which_oneof_stats == 'struct_stats':
common_stats = feature_stats.struct_stats.common_stats
else:
common_stats = feature_stats.string_stats.common_stats
assert num_examples >= common_stats.num_non_missing, (
'Total number of examples: {} is less than number of non missing '
'examples: {} for feature {}.'.format(
num_examples, common_stats.num_non_missing,
'.'.join(feature_stats.path.step)))
common_stats.num_missing = int(num_examples -
common_stats.num_non_missing)
if weighted_num_examples != 0:
common_stats.weighted_common_stats.num_missing = (
weighted_num_examples -
common_stats.weighted_common_stats.num_non_missing)
stats.num_examples = int(num_examples)