def _update_example_and_missing_count( stats: statistics_pb2.DatasetFeatureStatistics) -> None: """Updates example count of the dataset and missing count for all features.""" if not stats.features: return dummy_feature = stats_util.get_feature_stats(stats, _DUMMY_FEATURE_PATH) num_examples = stats_util.get_custom_stats(dummy_feature, _NUM_EXAMPLES_KEY) weighted_num_examples = stats_util.get_custom_stats( dummy_feature, _WEIGHTED_NUM_EXAMPLES_KEY) stats.features.remove(dummy_feature) for feature_stats in stats.features: # For features nested under a STRUCT feature, their num_missing is computed # in the basic stats generator (because their num_missing is relative to # their parent's value count). if len(feature_stats.path.step) > 1: continue common_stats = None which_oneof_stats = feature_stats.WhichOneof('stats') if which_oneof_stats is None: # There are not common_stats for this feature (which can be the case when # generating only custom_stats for a sparse or weighted feature). In that # case, simply continue without modifying the common stats. continue common_stats = getattr(feature_stats, which_oneof_stats).common_stats assert num_examples >= common_stats.num_non_missing, ( 'Total number of examples: {} is less than number of non missing ' 'examples: {} for feature {}.'.format( num_examples, common_stats.num_non_missing, '.'.join(feature_stats.path.step))) num_missing = int(num_examples - common_stats.num_non_missing) common_stats.num_missing = num_missing if common_stats.presence_and_valency_stats: common_stats.presence_and_valency_stats[ 0].num_missing = num_missing if weighted_num_examples != 0: weighted_num_missing = ( weighted_num_examples - common_stats.weighted_common_stats.num_non_missing) common_stats.weighted_common_stats.num_missing = weighted_num_missing if common_stats.weighted_presence_and_valency_stats: common_stats.weighted_presence_and_valency_stats[ 0].num_missing = (weighted_num_missing) stats.num_examples = int(num_examples) stats.weighted_num_examples = weighted_num_examples
def lists_to_partitions( datasets: DatasetFeatureStatisticsList, schema: Schema, examples: types.Artifact, partitions: List[List[Text]], ) -> List[Partition]: result: List[Partition] = [] for p in partitions: name = '__'.join(p) partition = Partition( name=name, statistics=DatasetFeatureStatisticsList(datasets=[ DatasetFeatureStatistics( name=name, num_examples=min([ getattr(feature, feature.WhichOneof( 'stats')).common_stats.num_non_missing for feature in filter(lambda f: feature_name(f) in p, dataset.features) ]), weighted_num_examples=0, features=list( filter(lambda f: feature_name(f) in p, dataset.features)), ) for dataset in datasets.datasets ]), example_splits=[ ExampleSplit(split=split, uri=os.path.join(examples.uri, split)) for split in artifact_utils.decode_split_names(examples.split_names) ], schema=Schema(feature=filter(lambda f: f.name in p, schema.feature), sparse_feature=filter(lambda f: f.name in p, schema.sparse_feature), weighted_feature=filter(lambda f: f.name in p, schema.weighted_feature), string_domain=filter(lambda f: f.name in p, schema.string_domain), float_domain=filter(lambda f: f.name in p, schema.float_domain), int_domain=filter(lambda f: f.name in p, schema.int_domain), default_environment=schema.default_environment)) result.append(partition) return result
def _update_example_and_missing_count( stats: statistics_pb2.DatasetFeatureStatistics) -> None: """Updates example count of the dataset and missing count for all features.""" if not stats.features: return dummy_feature = stats_util.get_feature_stats(stats, _DUMMY_FEATURE_PATH) num_examples = stats_util.get_custom_stats(dummy_feature, _NUM_EXAMPLES_KEY) weighted_num_examples = stats_util.get_custom_stats( dummy_feature, _WEIGHTED_NUM_EXAMPLES_KEY) stats.features.remove(dummy_feature) for feature_stats in stats.features: # For features nested under a STRUCT feature, their num_missing is computed # in the basic stats generator (because their num_missing is relative to # their parent's value count). if len(feature_stats.path.step) > 1: continue common_stats = None which_oneof_stats = feature_stats.WhichOneof('stats') if which_oneof_stats == 'num_stats': common_stats = feature_stats.num_stats.common_stats elif which_oneof_stats == 'struct_stats': common_stats = feature_stats.struct_stats.common_stats else: common_stats = feature_stats.string_stats.common_stats assert num_examples >= common_stats.num_non_missing, ( 'Total number of examples: {} is less than number of non missing ' 'examples: {} for feature {}.'.format( num_examples, common_stats.num_non_missing, '.'.join(feature_stats.path.step))) common_stats.num_missing = int(num_examples - common_stats.num_non_missing) if weighted_num_examples != 0: common_stats.weighted_common_stats.num_missing = ( weighted_num_examples - common_stats.weighted_common_stats.num_non_missing) stats.num_examples = int(num_examples)