def test_lock_till_released_failed(cluster: Cluster, runtime: DockerImage,
payload: list, signature: ModelSignature):
current_dir = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(current_dir, 'resources/identity_model/')
model_version_builder = ModelVersionBuilder(config.default_model_name, model_path) \
.with_runtime(runtime) \
.with_payload(payload) \
.with_signature(signature) \
.with_install_command("exit 1")
mv: ModelVersion = model_version_builder.build(cluster)
with pytest.raises(ModelVersion.ReleaseFailed):
mv.lock_till_released(timeout=config.lock_timeout)
def create_model_version_builder(name: str):
current_dir = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(current_dir, 'resources/identity_model/')
return ModelVersionBuilder(name, model_path) \
.with_runtime(runtime) \
.with_payload(payload) \
.with_signature(signature)
def app_scalar(cluster: Cluster, model_version_builder: ModelVersionBuilder,
training_data: str):
model_version_builder.with_monitoring_configuration(
MonitoringConfiguration(batch_size=10))
mv: ModelVersion = model_version_builder.build(cluster)
mv.training_data = training_data
data_upload_response = mv.upload_training_data()
data_upload_response.wait(sleep=5)
mv.lock_till_released(timeout=config.lock_timeout)
stage = ExecutionStageBuilder().with_model_variant(mv, 100).build()
app: Application = ApplicationBuilder(f"{config.default_application_name}-{random.randint(0, 1e5)}") \
.with_stage(stage).build(cluster)
app.lock_while_starting(timeout=config.lock_timeout)
time.sleep(10)
yield app
Application.delete(cluster, app.name)
def servable_tensor(cluster: Cluster, tensor_model_version_builder: ModelVersionBuilder):
mv: ModelVersion = tensor_model_version_builder.build(cluster)
mv.lock_till_released(timeout=config.lock_timeout)
sv: Servable = Servable.create(cluster, mv.name, mv.version)
sv.lock_while_starting(timeout=config.lock_timeout)
yield sv
Servable.delete(cluster, sv.name)
def model_version_builder(payload, signature, runtime, monitoring_configuration) -> ModelVersionBuilder:
current_dir = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(current_dir, 'resources/identity_model/')
return ModelVersionBuilder(config.default_model_name, model_path) \
.with_runtime(runtime) \
.with_signature(signature) \
.with_payload(payload) \
.with_monitoring_configuration(monitoring_configuration)
def test_build_logs_not_empty(cluster: Cluster,
model_version_builder: ModelVersionBuilder):
mv: ModelVersion = model_version_builder.build(cluster)
mv.lock_till_released(timeout=config.lock_timeout)
i = 0
for _ in mv.build_logs():
i += 1
assert i > 0
def test_training_data_upload(cluster: Cluster,
model_version_builder: ModelVersionBuilder,
training_data: str):
mv: ModelVersion = model_version_builder.build(cluster)
mv.training_data = training_data
data_upload_response = mv.upload_training_data()
data_upload_response.wait(sleep=5)
resp = cluster.request("GET", f"/monitoring/profiles/batch/{mv.id}/status")
assert "Success" == resp.json()["kind"]
def test_model_version_builder_build(cluster: Cluster):
name = config.default_model_name
current_dir = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(current_dir, 'resources/identity_model/')
signature = SignatureBuilder('infer') \
.with_input('in1', 'double', [-1, 2], ProfilingType.NONE) \
.with_output('out1', 'double', [-1], ProfilingType.NONE).build()
batch_size = 10
model_version_builder = ModelVersionBuilder(name, model_path) \
.with_runtime(config.runtime) \
.with_payload(['./src/func_main.py']) \
.with_signature(signature) \
.with_metadata({"key": "value"}) \
.with_monitoring_configuration(MonitoringConfiguration(batch_size=batch_size))
mv: ModelVersion = model_version_builder.build(cluster)
assert mv.status is ModelVersionStatus.Assembling
mv.lock_till_released(timeout=config.lock_timeout)
assert mv.status is ModelVersionStatus.Released
assert mv.monitoring_configuration.batch_size == batch_size
def tensor_model_version_builder(payload, runtime, monitoring_configuration) -> ModelVersionBuilder:
current_dir = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(current_dir, 'resources/identity_model/')
signature = SignatureBuilder('infer') \
.with_input('input', 'int64', [1], ProfilingType.NONE) \
.with_output('output', 'int64', [1], ProfilingType.NONE).build()
return ModelVersionBuilder(config.default_model_name, model_path) \
.with_runtime(runtime) \
.with_signature(signature) \
.with_payload(payload) \
.with_monitoring_configuration(monitoring_configuration)
def app_tensor(cluster: Cluster,
tensor_model_version_builder: ModelVersionBuilder):
mv: ModelVersion = tensor_model_version_builder.build(cluster)
mv.lock_till_released(timeout=config.lock_timeout)
stage = ExecutionStageBuilder().with_model_variant(mv, 100).build()
app = ApplicationBuilder(f"{config.default_application_name}-{random.randint(0, 1e5)}") \
.with_stage(stage).build(cluster)
app.lock_while_starting(timeout=config.lock_timeout)
time.sleep(10)
yield app
Application.delete(cluster, app.name)
def test_model_absolute_training_data():
name = config.default_model_name
path = "/home/user/folder/model/cool/"
training_data = "/home/data/training-data.csv"
try:
b = ModelVersionBuilder(name, path) \
.with_training_data(training_data)
pytest.fail(f"Should be unreachable: {b.training_data}")
except FileNotFoundError as err:
assert str(
err
) == "Can't find training data file /home/data/training-data.csv"
def test_ModelField_contact_signature_name_none():
name = config.default_model_name
path = "/home/user/folder/model/cool/"
signature = ModelSignature(inputs=[
ModelField(name="test", dtype=DataType.Name(2), shape=TensorShape())
],
outputs=[
ModelField(name="test",
dtype=DataType.Name(2),
shape=TensorShape())
])
with pytest.raises(
SignatureViolationException,
match=r"Creating model without signature_name is not allowed.*"):
_ = ModelVersionBuilder(name, path).with_signature(signature)
def test_delete(cluster: Cluster, model_version_builder: ModelVersionBuilder,
monitoring_mv: ModelVersion):
new_root_mv: ModelVersion = model_version_builder.build(cluster)
ms_config = MetricSpecConfig(monitoring_mv.id, 10, ThresholdCmpOp.NOT_EQ)
ms1 = MetricSpec.create(cluster, "test_list_for_modelversion",
new_root_mv.id, ms_config)
ms2 = MetricSpec.create(cluster, "test_list_for_modelversion",
new_root_mv.id, ms_config)
MetricSpec.delete(cluster, ms2.id)
mss = MetricSpec.find_by_modelversion(cluster, new_root_mv.id)
try:
assert len(mss) == 1
assert ms1.id in [ms.id for ms in mss]
assert ms2.id not in [ms.id for ms in mss]
finally:
MetricSpec.delete(cluster, ms1.id)
def test_ModelField_dt_invalid_output():
name = config.default_model_name
path = "/home/user/folder/model/cool/"
signature = ModelSignature(
signature_name="test",
inputs=[
ModelField(name="test",
dtype=DataType.Name(2),
shape=TensorShape())
],
outputs=[ModelField(name="test", shape=TensorShape())])
with pytest.raises(
SignatureViolationException,
match=r"Creating model with invalid dtype in the signature output.*"
):
_ = ModelVersionBuilder(name, path).with_signature(signature)
def test_create_high_level(cluster: Cluster,
model_version_builder: ModelVersionBuilder,
monitoring_mv: ModelVersion):
root_mv: ModelVersion = model_version_builder.build(cluster)
root_mv.lock_till_released(timeout=config.lock_timeout)
metric = monitoring_mv.as_metric(10, ThresholdCmpOp.NOT_EQ)
root_mv.assign_metrics([metric])
try:
assert monitoring_mv.name in [
ms.name
for ms in MetricSpec.find_by_modelversion(cluster, root_mv.id)
]
finally:
for ms in MetricSpec.find_by_modelversion(cluster, root_mv.id):
MetricSpec.delete(cluster, ms.id)
def test_model_create_programmatically():
name = config.default_model_name
runtime = config.runtime
path = "/home/user/folder/model/cool/"
payload = [
'./src/func_main.py', './requirements.txt', './data/*',
'./model/snapshot.proto'
]
resolved_payload = [
'/home/user/folder/model/cool/src/func_main.py',
'/home/user/folder/model/cool/requirements.txt',
'/home/user/folder/model/cool/data/*',
'/home/user/folder/model/cool/model/snapshot.proto',
]
signature = SignatureBuilder('infer') \
.with_input('in1', 'double', [-1, 2], ProfilingType.NUMERICAL) \
.with_output('out1', 'double', [-1], ProfilingType.NUMERICAL).build()
monitoring_configuration = MonitoringConfiguration(batch_size=10)
metadata = {"key": "value"}
install_command = "pip install -r requirements.txt"
training_data = "s3://bucket/path/to/training-data"
model_version_builder = ModelVersionBuilder(name, path) \
.with_runtime(runtime) \
.with_payload(payload) \
.with_signature(signature) \
.with_metadata(metadata) \
.with_install_command(install_command) \
.with_training_data(training_data) \
.with_monitoring_configuration(monitoring_configuration)
assert model_version_builder.name == name
assert model_version_builder.runtime == runtime
assert model_version_builder.path == path
assert list(model_version_builder.payload.values()) == payload
assert list(model_version_builder.payload.keys()) == resolved_payload
assert model_version_builder.signature == signature
assert model_version_builder.monitoring_configuration == monitoring_configuration
def test_modelversion_find(cluster: Cluster,
model_version_builder: ModelVersionBuilder):
mv: ModelVersion = model_version_builder.build(cluster)
mv_found: ModelVersion = ModelVersion.find(cluster, mv.name, mv.version)
assert mv.id == mv_found.id
def test_model_create_signature_validation():
name = config.default_model_name
path = "/home/user/folder/model/cool/"
signature = SignatureBuilder('infer').build()
with pytest.raises(SignatureViolationException):
ModelVersionBuilder(name, path).with_signature(signature)
def test_modelversion_list(cluster: Cluster,
model_version_builder: ModelVersionBuilder):
mv: ModelVersion = model_version_builder.build(cluster)
assert mv.id in [
modelversion.id for modelversion in ModelVersion.list(cluster)
]
def root_mv(cluster: Cluster, model_version_builder: ModelVersionBuilder):
mv: ModelVersion = model_version_builder.build(cluster)
mv.lock_till_released(timeout=config.lock_timeout)
return mv
def apply(self, conn: Cluster, cwd):
mv_builder = ModelVersionBuilder(name = self.name,path = cwd) \
.with_runtime(DockerImage.from_string(self.runtime)) \
.with_payload(self.payload) \
.with_signature(self.contract.to_proto())
if self.install_command:
mv_builder.with_install_command(self.install_command)
if self.training_data:
mv_builder.with_training_data(self.training_data)
collected_meta = CollectedMetadata.collect(cwd).to_metadata()
if self.metadata:
collected_meta.update(self.metadata)
mv_builder.with_metadata(collected_meta)
if self.monitoring_configuration:
mc = SDK_MC(self.monitoring_configuration.batch_size)
mv_builder.with_monitoring_configuration(mc)
logging.debug(f"Model version builder:\n{mv_builder}")
mv = mv_builder.build(conn)
build_log_handler = DockerLogHandler()
logging.info("Build logs:")
for ev in mv.build_logs():
build_log_handler.show(ev.data)
if self.monitoring:
logging.info(
f"Uploading monitoring configuration for the model {mv.name}:{mv.version}"
)
for mon in self.monitoring:
name, version = mon.config.monitoring_model.split(":")
mon_mv = SDK_MV.find(conn, name, int(version))
sdk_conf = MetricSpecConfig(modelversion_id=mon_mv.id,
threshold=mon.config.threshold,
threshold_op=mon.config.operator)
sdk_ms = MetricSpec.create(cluster=conn,
name=mon.name,
modelversion_id=mv.id,
config=sdk_conf)
logging.debug(
f"Created metric spec: {sdk_ms.name} with id {sdk_ms.id}")
if mv.training_data:
logging.info("Uploading training data")
resp = mv.upload_training_data()
logging.info(f"Training data profile is available at {resp.url}")
return mv