def assign_metrics(self, metrics: List[MetricModel], wait: bool = True):
"""
Adds metrics to the model.
:param metrics: list of metrics
:return: self
"""
if wait:
self.lock_till_released()
for metric in metrics:
modelversion = metric.modelversion
if wait:
modelversion.lock_till_released()
config = MetricSpecConfig(
modelversion_id=modelversion.id,
threshold=metric.threshold,
threshold_op=metric.comparator
)
MetricSpec.create(
cluster=self.cluster,
name=modelversion.name,
modelversion_id=self.id,
config=config
)
def test_list(cluster: Cluster, root_mv: ModelVersion,
monitoring_mv: ModelVersion):
ms_config = MetricSpecConfig(monitoring_mv.id, 10, ThresholdCmpOp.NOT_EQ)
ms = MetricSpec.create(cluster, "test_list", root_mv.id, ms_config)
try:
assert ms.name in [item.name for item in MetricSpec.list(cluster)]
finally:
MetricSpec.delete(cluster, ms.id)
def test_create_low_level(cluster: Cluster, root_mv: ModelVersion,
monitoring_mv: ModelVersion):
ms_config = MetricSpecConfig(monitoring_mv.id, 10, ThresholdCmpOp.NOT_EQ)
ms = MetricSpec.create(cluster, "test", root_mv.id, ms_config)
ms_found = MetricSpec.find_by_id(cluster, ms.id)
try:
assert ms_found.name == ms.name
assert ms_found.modelversion_id == ms.modelversion_id
finally:
MetricSpec.delete(cluster, ms.id)
def test_create_high_level(cluster: Cluster,
model_version_builder: ModelVersionBuilder,
monitoring_mv: ModelVersion):
root_mv: ModelVersion = model_version_builder.build(cluster)
root_mv.lock_till_released(timeout=config.lock_timeout)
metric = monitoring_mv.as_metric(10, ThresholdCmpOp.NOT_EQ)
root_mv.assign_metrics([metric])
try:
assert monitoring_mv.name in [
ms.name
for ms in MetricSpec.find_by_modelversion(cluster, root_mv.id)
]
finally:
for ms in MetricSpec.find_by_modelversion(cluster, root_mv.id):
MetricSpec.delete(cluster, ms.id)
def apply(self, conn: Cluster, cwd):
mv_builder = ModelVersionBuilder(name = self.name,path = cwd) \
.with_runtime(DockerImage.from_string(self.runtime)) \
.with_payload(self.payload) \
.with_signature(self.contract.to_proto())
if self.install_command:
mv_builder.with_install_command(self.install_command)
if self.training_data:
mv_builder.with_training_data(self.training_data)
collected_meta = CollectedMetadata.collect(cwd).to_metadata()
if self.metadata:
collected_meta.update(self.metadata)
mv_builder.with_metadata(collected_meta)
if self.monitoring_configuration:
mc = SDK_MC(self.monitoring_configuration.batch_size)
mv_builder.with_monitoring_configuration(mc)
logging.debug(f"Model version builder:\n{mv_builder}")
mv = mv_builder.build(conn)
build_log_handler = DockerLogHandler()
logging.info("Build logs:")
for ev in mv.build_logs():
build_log_handler.show(ev.data)
if self.monitoring:
logging.info(
f"Uploading monitoring configuration for the model {mv.name}:{mv.version}"
)
for mon in self.monitoring:
name, version = mon.config.monitoring_model.split(":")
mon_mv = SDK_MV.find(conn, name, int(version))
sdk_conf = MetricSpecConfig(modelversion_id=mon_mv.id,
threshold=mon.config.threshold,
threshold_op=mon.config.operator)
sdk_ms = MetricSpec.create(cluster=conn,
name=mon.name,
modelversion_id=mv.id,
config=sdk_conf)
logging.debug(
f"Created metric spec: {sdk_ms.name} with id {sdk_ms.id}")
if mv.training_data:
logging.info("Uploading training data")
resp = mv.upload_training_data()
logging.info(f"Training data profile is available at {resp.url}")
return mv
def test_delete(cluster: Cluster, model_version_builder: ModelVersionBuilder,
monitoring_mv: ModelVersion):
new_root_mv: ModelVersion = model_version_builder.build(cluster)
ms_config = MetricSpecConfig(monitoring_mv.id, 10, ThresholdCmpOp.NOT_EQ)
ms1 = MetricSpec.create(cluster, "test_list_for_modelversion",
new_root_mv.id, ms_config)
ms2 = MetricSpec.create(cluster, "test_list_for_modelversion",
new_root_mv.id, ms_config)
MetricSpec.delete(cluster, ms2.id)
mss = MetricSpec.find_by_modelversion(cluster, new_root_mv.id)
try:
assert len(mss) == 1
assert ms1.id in [ms.id for ms in mss]
assert ms2.id not in [ms.id for ms in mss]
finally:
MetricSpec.delete(cluster, ms1.id)