def __init__(self,
statistics: types.Channel = None,
schema: types.Channel = None,
exclude_splits: Optional[List[Text]] = None,
anomalies: Optional[Text] = None,
instance_name: Optional[Text] = None):
"""Construct an ExampleValidator component.
Args:
statistics: A Channel of type `standard_artifacts.ExampleStatistics`.
schema: A Channel of type `standard_artifacts.Schema`. _required_
exclude_splits: Names of splits that the example validator should not
validate. Default behavior (when exclude_splits is set to None)
is excluding no splits.
anomalies: Output channel of type `standard_artifacts.ExampleAnomalies`.
instance_name: Optional name assigned to this specific instance of
ExampleValidator. Required only if multiple ExampleValidator components
are declared in the same pipeline. Either `stats` or `statistics` must
be present in the arguments.
"""
if exclude_splits is None:
exclude_splits = []
logging.info('Excluding no splits because exclude_splits is not set.')
if not anomalies:
anomalies = types.Channel(type=standard_artifacts.ExampleAnomalies)
spec = ExampleValidatorSpec(
statistics=statistics,
schema=schema,
exclude_splits=json_utils.dumps(exclude_splits),
anomalies=anomalies)
super(ExampleValidator, self).__init__(
spec=spec, instance_name=instance_name)
def __init__(self,
stats: types.Channel = None,
schema: types.Channel = None,
output: Optional[types.Channel] = None,
statistics: Optional[types.Channel] = None,
instance_name: Optional[Text] = None):
"""Construct an ExampleValidator component.
Args:
stats: A Channel of 'ExampleStatisticsPath` type. This should contain at
least 'eval' split. Other splits are ignored currently. Will be
deprecated in the future for the `statistics` parameter.
schema: A Channel of "SchemaPath' type. _required_
output: Output channel of 'ExampleValidationPath' type.
statistics: Future replacement of the 'stats' argument.
instance_name: Optional name assigned to this specific instance of
ExampleValidator. Required only if multiple ExampleValidator components
are declared in the same pipeline.
Either `stats` or `statistics` must be present in the arguments.
"""
stats = stats or statistics
output = output or types.Channel(
type=standard_artifacts.ExampleAnomalies,
artifacts=[standard_artifacts.ExampleAnomalies()])
spec = ExampleValidatorSpec(stats=stats, schema=schema, output=output)
super(ExampleValidator, self).__init__(spec=spec,
instance_name=instance_name)
def __init__(self,
statistics: types.Channel = None,
schema: types.Channel = None,
output: Optional[types.Channel] = None,
stats: Optional[types.Channel] = None,
instance_name: Optional[Text] = None):
"""Construct an ExampleValidator component.
Args:
statistics: A Channel of type `standard_artifacts.ExampleStatistics`.
This should contain at least 'eval' split. Other splits are currently
ignored.
schema: A Channel of type `standard_artifacts.Schema`. _required_
output: Output channel of type `standard_artifacts.ExampleAnomalies`.
stats: Backwards compatibility alias for the 'statistics' argument.
instance_name: Optional name assigned to this specific instance of
ExampleValidator. Required only if multiple ExampleValidator components
are declared in the same pipeline.
Either `stats` or `statistics` must be present in the arguments.
"""
statistics = statistics or stats
output = output or types.Channel(
type=standard_artifacts.ExampleAnomalies,
artifacts=[standard_artifacts.ExampleAnomalies()])
spec = ExampleValidatorSpec(stats=statistics,
schema=schema,
output=output)
super(ExampleValidator, self).__init__(spec=spec,
instance_name=instance_name)
def __init__(self,
statistics: types.Channel = None,
schema: types.Channel = None,
output: Optional[types.Channel] = None,
stats: Optional[types.Channel] = None,
instance_name: Optional[Text] = None):
"""An ExampleValidator component for examples.
TFX has its ExampleValidator component, and this one uses the same
executor. The reason for this one to exist, is that the TFX
component does not allow to specify the splits to use, it just
assumes `train` and `eval`. This component will be unnecessary
once TFX Transform allows to set the input and output splits
as other components do"""
if stats:
logging.warning(
'The "stats" argument to the StatisticsGen component has '
'been renamed to "statistics" and is deprecated. Please update'
' your usage as support for this argument will be removed'
' soon.')
statistics = stats
anomalies = output or types.Channel(
type=standard_artifacts.ExampleAnomalies,
artifacts=[standard_artifacts.ExampleAnomalies()])
spec = ExampleValidatorSpec(statistics=statistics,
schema=schema,
anomalies=anomalies)
super(ExampleValidator, self).__init__(spec=spec,
instance_name=instance_name)
def __init__(self,
statistics: types.Channel = None,
schema: types.Channel = None,
exclude_splits: Optional[List[Text]] = None,
output: Optional[types.Channel] = None,
stats: Optional[types.Channel] = None,
instance_name: Optional[Text] = None):
"""Construct an ExampleValidator component.
Args:
statistics: A Channel of type `standard_artifacts.ExampleStatistics`. This
should contain at least 'eval' split. Other splits are currently
ignored.
schema: A Channel of type `standard_artifacts.Schema`. _required_
exclude_splits: Names of splits that the example validator should not
validate. Default behavior (when exclude_splits is set to None)
is excluding no splits.
output: Output channel of type `standard_artifacts.ExampleAnomalies`.
stats: Backwards compatibility alias for the 'statistics' argument.
instance_name: Optional name assigned to this specific instance of
ExampleValidator. Required only if multiple ExampleValidator components
are declared in the same pipeline. Either `stats` or `statistics` must
be present in the arguments.
"""
if stats:
logging.warning(
'The "stats" argument to the StatisticsGen component has '
'been renamed to "statistics" and is deprecated. Please update your '
'usage as support for this argument will be removed soon.')
statistics = stats
if exclude_splits is None:
exclude_splits = []
logging.info('Excluding no splits because exclude_splits is not set.')
anomalies = output
if not anomalies:
anomalies_artifact = standard_artifacts.ExampleAnomalies()
statistics_split_names = artifact_utils.decode_split_names(
artifact_utils.get_single_instance(list(
statistics.get())).split_names)
split_names = [
split for split in statistics_split_names
if split not in exclude_splits
]
anomalies_artifact.split_names = artifact_utils.encode_split_names(
split_names)
anomalies = types.Channel(
type=standard_artifacts.ExampleAnomalies,
artifacts=[anomalies_artifact])
spec = ExampleValidatorSpec(
statistics=statistics,
schema=schema,
exclude_splits=json_utils.dumps(exclude_splits),
anomalies=anomalies)
super(ExampleValidator, self).__init__(
spec=spec, instance_name=instance_name)
def __init__(self,
stats: types.Channel,
schema: types.Channel,
output: Optional[types.Channel] = None,
name: Optional[Text] = None):
"""Construct an ExampleValidator component.
Args:
stats: A Channel of 'ExampleStatisticsPath' type. This should contain at
least 'eval' split. Other splits are ignored currently.
schema: A Channel of "SchemaPath' type.
output: Optional output channel of 'ExampleValidationPath' type.
name: Optional unique name. Necessary iff multiple ExampleValidator
components are declared in the same pipeline.
"""
output = output or types.Channel(
type=standard_artifacts.ExampleAnomalies,
artifacts=[standard_artifacts.ExampleAnomalies()])
spec = ExampleValidatorSpec(stats=stats, schema=schema, output=output)
super(ExampleValidator, self).__init__(spec=spec, name=name)
def __init__(self,
statistics: types.Channel = None,
schema: types.Channel = None,
output: Optional[types.Channel] = None,
stats: Optional[types.Channel] = None,
instance_name: Optional[Text] = None,
enable_cache: Optional[bool] = None):
"""Construct an ExampleValidator component.
Args:
statistics: A Channel of type `standard_artifacts.ExampleStatistics`. This
should contain at least 'eval' split. Other splits are currently
ignored.
schema: A Channel of type `standard_artifacts.Schema`. _required_
output: Output channel of type `standard_artifacts.ExampleAnomalies`.
stats: Backwards compatibility alias for the 'statistics' argument.
instance_name: Optional name assigned to this specific instance of
ExampleValidator. Required only if multiple ExampleValidator components
are declared in the same pipeline. Either `stats` or `statistics` must
be present in the arguments.
enable_cache: Optional boolean to indicate if cache is enabled for the
ExampleValidator component. If not specified, defaults to the value
specified for pipeline's enable_cache parameter.
"""
if stats:
absl.logging.warning(
'The "stats" argument to the StatisticsGen component has '
'been renamed to "statistics" and is deprecated. Please update your '
'usage as support for this argument will be removed soon.')
statistics = stats
anomalies = output or types.Channel(
type=standard_artifacts.ExampleAnomalies,
artifacts=[standard_artifacts.ExampleAnomalies()])
spec = ExampleValidatorSpec(statistics=statistics,
schema=schema,
anomalies=anomalies)
super(ExampleValidator, self).__init__(spec=spec,
instance_name=instance_name,
enable_cache=enable_cache)
def __init__(self,
statistics: types.Channel = None,
schema: types.Channel = None,
exclude_splits: Optional[List[Text]] = None):
"""Construct an ExampleValidator component.
Args:
statistics: A Channel of type `standard_artifacts.ExampleStatistics`.
schema: A Channel of type `standard_artifacts.Schema`. _required_
exclude_splits: Names of splits that the example validator should not
validate. Default behavior (when exclude_splits is set to None)
is excluding no splits.
"""
if exclude_splits is None:
exclude_splits = []
logging.info(
'Excluding no splits because exclude_splits is not set.')
anomalies = types.Channel(type=standard_artifacts.ExampleAnomalies)
spec = ExampleValidatorSpec(
statistics=statistics,
schema=schema,
exclude_splits=json_utils.dumps(exclude_splits),
anomalies=anomalies)
super(ExampleValidator, self).__init__(spec=spec)
