def _scaffold_model_card(self) -> ModelCard:
"""Generates the model card during scaffold_assets phase.
It includes the implementation details for auto-populated ModelCard fields
given the specialization of the ModelCardToolkit.
Returns:
A ModelCard representing the given model.
"""
model_card = ModelCard()
if self._store:
model_card = tfx_util.generate_model_card_for_model(
self._store, self._artifact_with_model_uri.id)
metrics_artifacts = tfx_util.get_metrics_artifacts_for_model(
self._store, self._artifact_with_model_uri.id)
stats_artifacts = tfx_util.get_stats_artifacts_for_model(
self._store, self._artifact_with_model_uri.id)
for metrics_artifact in metrics_artifacts:
eval_result = tfx_util.read_metrics_eval_result(
metrics_artifact.uri)
if eval_result is not None:
graphics.annotate_eval_result_plots(
model_card, eval_result)
for stats_artifact in stats_artifacts:
train_stats = tfx_util.read_stats_proto(
stats_artifact.uri, 'train')
eval_stats = tfx_util.read_stats_proto(stats_artifact.uri,
'eval')
graphics.annotate_dataset_feature_statistics_plots(
model_card, [train_stats, eval_stats])
return model_card
def scaffold_assets(self) -> ModelCard:
"""Generates the model cards tookit assets.
Model cards assets include the model card json file and customizable model
card UI templates.
An assets directory is created if one does not already exist.
If the MCT is initialized with a `mlmd_store`, it further auto-populates
the model cards properties as well as generating related plots such as model
performance and data distributions.
Returns:
A ModelCard representing the given model.
Raises:
FileNotFoundError: if it failed to copy the UI template files.
"""
model_card = ModelCard()
if self._store:
model_card = tfx_util.generate_model_card_for_model(
self._store, self._artifact_with_model_uri.id)
metrics_artifacts = tfx_util.get_metrics_artifacts_for_model(
self._store, self._artifact_with_model_uri.id)
stats_artifacts = tfx_util.get_stats_artifacts_for_model(
self._store, self._artifact_with_model_uri.id)
for metrics_artifact in metrics_artifacts:
eval_result = tfx_util.read_metrics_eval_result(
metrics_artifact.uri)
if eval_result is not None:
graphics.annotate_eval_result_plots(
model_card, eval_result)
for stats_artifact in stats_artifacts:
train_stats = tfx_util.read_stats_proto(
stats_artifact.uri, 'train')
eval_stats = tfx_util.read_stats_proto(stats_artifact.uri,
'eval')
graphics.annotate_dataset_feature_statistics_plots(
model_card, train_stats, eval_stats)
# Write JSON file.
self._write_file(self._mcta_json_file, model_card.to_json())
# Write UI template files.
for template_path in _UI_TEMPLATES:
template_content = pkgutil.get_data('model_card_toolkit',
template_path)
if template_content is None:
raise FileNotFoundError(f"Cannot find file: '{template_path}'")
template_content = template_content.decode('utf8')
self._write_file(os.path.join(self.output_dir, template_path),
template_content)
return model_card
def test_generate_model_card_for_model(self):
store = testdata_utils.get_tfx_pipeline_metadata_store(self.tmp_db_path)
model_card = tfx_util.generate_model_card_for_model(
store, testdata_utils.TFX_0_21_MODEL_ARTIFACT_ID)
trainers = store.get_executions_by_id([testdata_utils.TFX_0_21_TRAINER_ID])
self.assertNotEmpty(trainers)
model_details = model_card.model_details
self.assertEqual(model_details.name,
trainers[-1].properties['module_file'].string_value)
self.assertEqual(model_details.version.name,
trainers[-1].properties['checksum_md5'].string_value)
self.assertIn(
trainers[-1].properties['pipeline_name'].string_value,
[reference.reference for reference in model_details.references])
datasets = store.get_artifacts_by_id(
[testdata_utils.TFX_0_21_MODEL_DATASET_ID])
self.assertNotEmpty(datasets)
def scaffold_assets(self) -> model_card_module.ModelCard:
"""Generates the model cards tookit assets.
Model cards assets include the model card json file and customizable model
card UI templates.
An assets directory is created if one does not already exist.
If the MCT is initialized with a `mlmd_store`, it further auto-populates
the model cards properties as well as generating related plots such as model
performance and data distributions.
Returns:
A ModelCard representing the given model.
"""
model_card = model_card_module.ModelCard()
if self._store:
model_card = tfx_util.generate_model_card_for_model(
self._store, self._artifact_with_model_uri.id)
metrics_artifacts = tfx_util.get_metrics_artifacts_for_model(
self._store, self._artifact_with_model_uri.id)
stats_artifacts = tfx_util.get_stats_artifacts_for_model(
self._store, self._artifact_with_model_uri.id)
for metrics_artifact in metrics_artifacts:
eval_result = tfx_util.read_metrics_eval_result(
metrics_artifact.uri)
if eval_result is not None:
graphics.annotate_eval_result_plots(
model_card, eval_result)
for stats_artifact in stats_artifacts:
train_stats = tfx_util.read_stats_proto(
stats_artifact.uri, 'train')
eval_stats = tfx_util.read_stats_proto(stats_artifact.uri,
'eval')
graphics.annotate_dataset_feature_statistics_plots(
model_card, train_stats, eval_stats)
# Write JSON file.
self._write_file(self._mcta_json_file, model_card.to_json())
# Write UI template files.
shutil.copytree(_UI_TEMPLATE_DIR, self._mcta_template_dir)
return model_card
def _scaffold_model_card(self) -> ModelCard:
"""Generates the ModelCard for scaffold_assets().
If Source is provided, pre-populate ModelCard fields with data from Source.
If MLMD store is provided, pre-populate ModelCard fields with data from
MLMD. See `model_card_toolkit.utils.tfx_util` and
`model_card_toolkit.utils.graphics` documentation for more details.
Returns:
A ModelCard representing the given model.
"""
# Pre-populate ModelCard fields
if self._store:
model_card = tfx_util.generate_model_card_for_model(
self._store, self._artifact_with_model_uri.id)
else:
model_card = ModelCard()
model_card = self._annotate_eval_results(model_card)
model_card = self._annotate_dataset_statistics(model_card)
model_card = self._annotate_model(model_card)
return model_card
def test_generate_model_card_for_model_with_invalid_db(self):
empty_db = self._get_empty_metadata_store()
with self.assertRaisesRegex(ValueError, '`store` is invalid'):
tfx_util.generate_model_card_for_model(
empty_db, testdata_utils.TFX_0_21_MODEL_ARTIFACT_ID)
def test_generate_model_card_for_model_with_invalid_model(self):
store = testdata_utils.get_tfx_pipeline_metadata_store(self.tmp_db_path)
with self.assertRaisesRegex(ValueError, 'not an instance of Model'):
tfx_util.generate_model_card_for_model(
store, testdata_utils.TFX_0_21_MODEL_DATASET_ID)
def test_generate_model_card_for_model_with_model_not_found(self):
store = testdata_utils.get_tfx_pipeline_metadata_store(self.tmp_db_path)
with self.assertRaisesRegex(ValueError, 'model_id cannot be found'):
model = metadata_store_pb2.Artifact()
tfx_util.generate_model_card_for_model(store, model.id)
