def test_log_metadata_successfully(self):
ws1 = metadata.Workspace(backend_url_prefix="127.0.0.1:8080",
name="ws_1",
description="a workspace for testing",
labels={"n1": "v1"})
r = metadata.Run(
workspace=ws1,
name="first run",
description="first run in ws_1",
)
data_set = r.log(
metadata.DataSet(description="an example data",
name="mytable-dump",
owner="*****@*****.**",
uri="file://path/to/dataset",
version="v1.0.0",
query="SELECT * FROM mytable"))
assert data_set.id
metrics = r.log(
metadata.Metrics(
name="MNIST-evaluation",
description=
"validating the MNIST model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist-eval.csv",
data_set_id="123",
model_id="12345",
metrics_type=metadata.Metrics.VALIDATION,
values={"accuracy": 0.95},
labels={"mylabel": "l1"}))
assert metrics.id
model = r.log(
metadata.Model(name="MNIST",
description="model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist",
model_type="neural network",
training_framework={
"name": "tensorflow",
"version": "v1.0"
},
hyperparameters={
"learning_rate": 0.5,
"layers": [10, 3, 1],
"early_stop": True
},
version="v0.0.1",
labels={"mylabel": "l1"}))
assert model.id
self.assertTrue(len(ws1.list()) > 0)
self.assertTrue(len(ws1.list(metadata.Model.ARTIFACT_TYPE_NAME)) > 0)
self.assertTrue(len(ws1.list(metadata.Metrics.ARTIFACT_TYPE_NAME)) > 0)
self.assertTrue(len(ws1.list(metadata.DataSet.ARTIFACT_TYPE_NAME)) > 0)
def log_model_info(ws, ws_run, model_uri):
exec2 = metadata.Execution(
name="execution" + datetime.utcnow().isoformat("T"),
workspace=ws,
run=ws_run,
description="train action",
)
_ = exec2.log_input(
metadata.Model(description="t2t model",
name="t2t-model",
owner="*****@*****.**",
uri=model_uri,
version="v1.0.0"))
def log_model(self,
name,
framework_dict,
hyper_param_dict,
desc="",
model_file_uri="",
model_type=""):
self._model = self._exec.log_output(
metadata.Model(name=name,
description=desc,
owner=self._owner,
uri=model_file_uri,
model_type=model_type,
training_framework=framework_dict,
hyperparameters=hyper_param_dict,
version=datetime.utcnow().isoformat("T")))
return self._model
def test_log_metadata_successfully_with_minimum_information(self):
ws1 = metadata.Workspace(backend_url_prefix="127.0.0.1:8080", name="ws_1")
r = metadata.Run(workspace=ws1, name="first run")
e = metadata.Execution(name="test execution", workspace=ws1, run=r)
self.assertIsNotNone(e.id)
data_set = e.log_input(
metadata.DataSet(name="mytable-dump", uri="file://path/to/dataset"))
self.assertIsNotNone(data_set.id)
metrics = e.log_output(
metadata.Metrics(name="MNIST-evaluation",
uri="gcs://my-bucket/mnist-eval.csv"))
self.assertIsNotNone(metrics.id)
model = e.log_output(
metadata.Model(name="MNIST", uri="gcs://my-bucket/mnist"))
self.assertIsNotNone(model.id)
validation_split=val_per)
print('Done training!')
# ## 5.1 Log model and metrics
# In[20]:
logmodel = exec.log_output(
metadata.Model(name="DeepCollaborativeFiltering",
description="Model for product recommender",
uri="",
model_type="neural network",
version=execTime,
training_framework={
"name": "tensorflow",
"version": "v1.14"
},
hyperparameters={
"batch_size": 64,
"validation_split": 0.25,
"layers": [n_customers, n_products, n_factors],
"epochs": 3
}))
metrics = exec.log_output(
metadata.Metrics(name="Model for product recommender evaluation",
description="Validating of the recommender model",
uri="",
version=execTime,
data_set_id=data_set.id,
model_id=logmodel.id))
def test_log_metadata_successfully(self):
ws1 = metadata.Workspace(
backend_url_prefix="127.0.0.1:8080",
name="ws_1",
description="a workspace for testing",
labels={"n1": "v1"})
r = metadata.Run(
workspace=ws1,
name="first run",
description="first run in ws_1",
)
e = metadata.Execution(
name="test execution",
workspace=ws1,
run=r,
description="an execution",
)
self.assertIsNotNone(e.id)
data_set = e.log_input(
metadata.DataSet(
description="an example data",
name="mytable-dump",
owner="*****@*****.**",
uri="file://path/to/dataset",
version="v1.0.0",
query="SELECT * FROM mytable"))
self.assertIsNotNone(data_set.id)
metrics = e.log_output(
metadata.Metrics(
name="MNIST-evaluation",
description="validating the MNIST model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist-eval.csv",
data_set_id="123",
model_id="12345",
metrics_type=metadata.Metrics.VALIDATION,
values={"accuracy": 0.95},
labels={"mylabel": "l1"}))
self.assertIsNotNone(metrics.id)
model = e.log_output(
metadata.Model(
name="MNIST",
description="model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist",
model_type="neural network",
training_framework={
"name": "tensorflow",
"version": "v1.0"
},
hyperparameters={
"learning_rate": 0.5,
"layers": [10, 3, 1],
"early_stop": True
},
version="v0.0.1",
labels={"mylabel": "l1"}))
self.assertIsNotNone(model.id)
# Test listing artifacts in a workspace
self.assertTrue(len(ws1.list()) > 0)
self.assertTrue(len(ws1.list(metadata.Model.ARTIFACT_TYPE_NAME)) > 0)
self.assertTrue(len(ws1.list(metadata.Metrics.ARTIFACT_TYPE_NAME)) > 0)
self.assertTrue(len(ws1.list(metadata.DataSet.ARTIFACT_TYPE_NAME)) > 0)
# Test lineage tracking.
output_events = ws1.client.list_events2(model.id).events
assert len(output_events) == 1
execution_id = output_events[0].execution_id
assert execution_id == e.id
all_events = ws1.client.list_events(execution_id).events
assert len(all_events) == 3
data_set = exec.log_input(
metadata.DataSet(description="an example data",
name="mytable-dump",
owner="*****@*****.**",
uri="file://path/to/dataset",
version="v1.0.0",
query="SELECT * FROM mytable"))
model = exec.log_output(
metadata.Model(name="MNIST",
description="model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist",
model_type="neural network",
training_framework={
"name": "tensorflow",
"version": "v1.0"
},
hyperparameters={
"learning_rate": 0.5,
"layers": [10, 3, 1],
"early_stop": True
},
version="v0.0.1",
labels={"mylabel": "l1"}))
metrics = exec.log_output(
metadata.Metrics(
name="MNIST-evaluation",
description=
"validating the MNIST model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist-eval.csv",
data_set_id=data_set.id,