def testEnableCache(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator_1 = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model=channel_utils.as_channel([model_exports]))
self.assertEqual(None, evaluator_1.enable_cache)
evaluator_2 = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model=channel_utils.as_channel([model_exports]),
enable_cache=True)
self.assertEqual(True, evaluator_2.enable_cache)
def test_construct(self):
examples = types.Artifact(type_name='ExamplesPath')
model_exports = types.Artifact(type_name='ModelExportPath')
evaluator = component.Evaluator(
examples=channel.as_channel([examples]),
model_exports=channel.as_channel([model_exports]))
self.assertEqual('ModelEvalPath', evaluator.outputs.output.type_name)
def test_construct(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model_exports=channel_utils.as_channel([model_exports]))
self.assertEqual('ModelEvalPath', evaluator.outputs.output.type_name)
def testConstruct(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model=channel_utils.as_channel([model_exports]))
self.assertEqual(standard_artifacts.ModelEvaluation.TYPE_NAME,
evaluator.outputs['evaluation'].type_name)
def testConstructWithEvalConfig(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model_exports=channel_utils.as_channel([model_exports]),
eval_config=tfma.EvalConfig(
slicing_specs=[tfma.SlicingSpec(feature_keys=['trip_start_hour'])]))
self.assertEqual(standard_artifacts.ModelEvaluation.TYPE_NAME,
evaluator.outputs['output'].type_name)
def testConstructWithModuleFile(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model=channel_utils.as_channel([model_exports]),
example_splits=['eval'],
module_file='path')
self.assertEqual(standard_artifacts.ModelEvaluation.TYPE_NAME,
evaluator.outputs['evaluation'].type_name)
self.assertEqual('path', evaluator.exec_properties['module_file'])
def test_construct_with_slice_spec(self):
examples = types.Artifact(type_name='ExamplesPath')
model_exports = types.Artifact(type_name='ModelExportPath')
evaluator = component.Evaluator(
examples=channel.as_channel([examples]),
model_exports=channel.as_channel([model_exports]),
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
self.assertEqual('ModelEvalPath', evaluator.outputs.output.type_name)
def testConstructDuplicateUserModule(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
with self.assertRaises(ValueError):
_ = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model=channel_utils.as_channel([model_exports]),
example_splits=['eval'],
module_file='module_file_path',
module_path='python.path.module')
def testConstructWithSliceSpec(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model_exports=channel_utils.as_channel([model_exports]),
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]))
self.assertEqual('ModelEvalPath', evaluator.outputs.output.type_name)
def testConstructWithFairnessThresholds(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model_exports=channel_utils.as_channel([model_exports]),
feature_slicing_spec=evaluator_pb2.FeatureSlicingSpec(specs=[
evaluator_pb2.SingleSlicingSpec(
column_for_slicing=['trip_start_hour'])
]),
fairness_indicator_thresholds=[0.1, 0.3, 0.5, 0.9])
self.assertEqual('ModelEvalPath', evaluator.outputs['output'].type_name)
def testConstruct(self):
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model=channel_utils.as_channel([model_exports]),
example_splits=['eval'])
self.assertEqual(standard_artifacts.ModelEvaluation.TYPE_NAME,
evaluator.outputs['evaluation'].type_name)
self.assertEqual(standard_artifacts.ModelBlessing.TYPE_NAME,
evaluator.outputs['blessing'].type_name)
self.assertEqual(json_utils.dumps(['eval']),
evaluator.exec_properties['example_splits'])
def testConstructWithParameter(self):
column_name = data_types.RuntimeParameter(name='column-name', ptype=Text)
threshold = data_types.RuntimeParameter(name='threshold', ptype=float)
examples = standard_artifacts.Examples()
model_exports = standard_artifacts.Model()
evaluator = component.Evaluator(
examples=channel_utils.as_channel([examples]),
model=channel_utils.as_channel([model_exports]),
feature_slicing_spec={'specs': [{
'column_for_slicing': [column_name]
}]},
fairness_indicator_thresholds=[threshold])
self.assertEqual(standard_artifacts.ModelEvaluation.TYPE_NAME,
evaluator.outputs['evaluation'].type_name)
def __init__(self, examples: str, model_exports: str,
feature_slicing_spec: List[List[str]]):
slicing_spec = evaluator_pb2.FeatureSlicingSpec()
for slice_spec in feature_slicing_spec:
spec = slicing_spec.specs.add()
for column in slice_spec:
spec.column_for_slicing.append(column)
component = evaluator_component.Evaluator(
channel.Channel('ExamplesPath'),
channel.Channel('ModelExportPath'),
feature_slicing_spec=slicing_spec)
super().__init__(component, {
"examples": examples,
"model_exports": model_exports,
})