def test_log_metadata_successfully(self):
ws1 = metadata.Workspace(backend_url_prefix="127.0.0.1:8080",
name="ws_1",
description="a workspace for testing",
labels={"n1": "v1"})
r = metadata.Run(
workspace=ws1,
name="first run",
description="first run in ws_1",
)
data_set = r.log(
metadata.DataSet(description="an example data",
name="mytable-dump",
owner="*****@*****.**",
uri="file://path/to/dataset",
version="v1.0.0",
query="SELECT * FROM mytable"))
assert data_set.id
metrics = r.log(
metadata.Metrics(
name="MNIST-evaluation",
description=
"validating the MNIST model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist-eval.csv",
data_set_id="123",
model_id="12345",
metrics_type=metadata.Metrics.VALIDATION,
values={"accuracy": 0.95},
labels={"mylabel": "l1"}))
assert metrics.id
model = r.log(
metadata.Model(name="MNIST",
description="model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist",
model_type="neural network",
training_framework={
"name": "tensorflow",
"version": "v1.0"
},
hyperparameters={
"learning_rate": 0.5,
"layers": [10, 3, 1],
"early_stop": True
},
version="v0.0.1",
labels={"mylabel": "l1"}))
assert model.id
self.assertTrue(len(ws1.list()) > 0)
self.assertTrue(len(ws1.list(metadata.Model.ARTIFACT_TYPE_NAME)) > 0)
self.assertTrue(len(ws1.list(metadata.Metrics.ARTIFACT_TYPE_NAME)) > 0)
self.assertTrue(len(ws1.list(metadata.DataSet.ARTIFACT_TYPE_NAME)) > 0)
def log_metrics(self,
name,
metrics_dict,
desc="",
uri="gcs://path/to/metrics"):
metrics = self._exec.log_output(
metadata.Metrics(name=name,
owner=self._owner,
description=desc,
uri=uri,
model_id=self._model.id,
metrics_type=metadata.Metrics.VALIDATION,
values=metrics_dict))
return metrics
def test_log_metadata_successfully_with_minimum_information(self):
ws1 = metadata.Workspace(backend_url_prefix="127.0.0.1:8080", name="ws_1")
r = metadata.Run(workspace=ws1, name="first run")
e = metadata.Execution(name="test execution", workspace=ws1, run=r)
self.assertIsNotNone(e.id)
data_set = e.log_input(
metadata.DataSet(name="mytable-dump", uri="file://path/to/dataset"))
self.assertIsNotNone(data_set.id)
metrics = e.log_output(
metadata.Metrics(name="MNIST-evaluation",
uri="gcs://my-bucket/mnist-eval.csv"))
self.assertIsNotNone(metrics.id)
model = e.log_output(
metadata.Model(name="MNIST", uri="gcs://my-bucket/mnist"))
self.assertIsNotNone(model.id)
model_type="neural network",
version=execTime,
training_framework={
"name": "tensorflow",
"version": "v1.14"
},
hyperparameters={
"batch_size": 64,
"validation_split": 0.25,
"layers": [n_customers, n_products, n_factors],
"epochs": 3
}))
metrics = exec.log_output(
metadata.Metrics(name="Model for product recommender evaluation",
description="Validating of the recommender model",
uri="",
version=execTime,
data_set_id=data_set.id,
model_id=logmodel.id))
# # 6 Get current output directory for model
# In[21]:
directorystream = minioClient.get_object('data', 'recommender/directory.txt')
directory = ""
for d in directorystream.stream(32 * 1024):
directory += d.decode('utf-8')
arg_version = "1"
export_path = 's3://models/' + directory + '/' + arg_version + '/'
print('Exporting trained model to', export_path)
def test_log_metadata_successfully(self):
ws1 = metadata.Workspace(
backend_url_prefix="127.0.0.1:8080",
name="ws_1",
description="a workspace for testing",
labels={"n1": "v1"})
r = metadata.Run(
workspace=ws1,
name="first run",
description="first run in ws_1",
)
e = metadata.Execution(
name="test execution",
workspace=ws1,
run=r,
description="an execution",
)
self.assertIsNotNone(e.id)
data_set = e.log_input(
metadata.DataSet(
description="an example data",
name="mytable-dump",
owner="*****@*****.**",
uri="file://path/to/dataset",
version="v1.0.0",
query="SELECT * FROM mytable"))
self.assertIsNotNone(data_set.id)
metrics = e.log_output(
metadata.Metrics(
name="MNIST-evaluation",
description="validating the MNIST model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist-eval.csv",
data_set_id="123",
model_id="12345",
metrics_type=metadata.Metrics.VALIDATION,
values={"accuracy": 0.95},
labels={"mylabel": "l1"}))
self.assertIsNotNone(metrics.id)
model = e.log_output(
metadata.Model(
name="MNIST",
description="model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist",
model_type="neural network",
training_framework={
"name": "tensorflow",
"version": "v1.0"
},
hyperparameters={
"learning_rate": 0.5,
"layers": [10, 3, 1],
"early_stop": True
},
version="v0.0.1",
labels={"mylabel": "l1"}))
self.assertIsNotNone(model.id)
# Test listing artifacts in a workspace
self.assertTrue(len(ws1.list()) > 0)
self.assertTrue(len(ws1.list(metadata.Model.ARTIFACT_TYPE_NAME)) > 0)
self.assertTrue(len(ws1.list(metadata.Metrics.ARTIFACT_TYPE_NAME)) > 0)
self.assertTrue(len(ws1.list(metadata.DataSet.ARTIFACT_TYPE_NAME)) > 0)
# Test lineage tracking.
output_events = ws1.client.list_events2(model.id).events
assert len(output_events) == 1
execution_id = output_events[0].execution_id
assert execution_id == e.id
all_events = ws1.client.list_events(execution_id).events
assert len(all_events) == 3
"version": "v1.0"
},
hyperparameters={
"learning_rate": 0.5,
"layers": [10, 3, 1],
"early_stop": True
},
version="v0.0.1",
labels={"mylabel": "l1"}))
metrics = exec.log_output(
metadata.Metrics(
name="MNIST-evaluation",
description=
"validating the MNIST model to recognize handwritten digits",
owner="*****@*****.**",
uri="gcs://my-bucket/mnist-eval.csv",
data_set_id=data_set.id,
model_id=model.id,
metrics_type=metadata.Metrics.VALIDATION,
values={"accuracy": 0.95},
labels={"mylabel": "l1"}))
# List all the models in the workspace
# In[4]:
pandas.DataFrame.from_dict(ws1.list(metadata.Model.ARTIFACT_TYPE_NAME))
# Get basic lineage
# In[5]: