def test_predict_video_url_with_custom_sample_ms(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
video=resources_pb2.Video(url=BEER_VIDEO_URL)))
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(sample_ms=2000))),
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
# The expected time per frame is the middle between the start and the end of the frame
# (in milliseconds).
expected_time = 1000
assert len(response.outputs[0].data.frames) > 0
for frame in response.outputs[0].data.frames:
assert frame.frame_info.time == expected_time
expected_time += 2000
def test_predict_image_url_with_selected_concepts(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL, ), ), )
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(select_concepts=[
resources_pb2.Concept(name="dog"),
resources_pb2.Concept(name="cat"),
]))),
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
concepts = response.outputs[0].data.concepts
assert len(concepts) == 2
dog_concept = [c for c in concepts if c.name == "dog"][0]
cat_concept = [c for c in concepts if c.name == "cat"][0]
assert dog_concept.value > cat_concept.value
def test_predict_image_url(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL)))
],
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.concepts) > 0
def test_failed_predict(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=NON_EXISTING_IMAGE_URL)))
],
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
assert response.status.code == status_code_pb2.FAILURE
assert response.status.description == "Failure"
assert response.outputs[
0].status.code == status_code_pb2.INPUT_DOWNLOAD_FAILED
def test_image_predict_on_public_models(channel):
stub = service_pb2_grpc.V2Stub(channel)
for title, model_id in MODEL_TITLE_AND_ID_PAIRS:
request = service_pb2.PostModelOutputsRequest(
model_id=model_id,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL)))
],
)
response = post_model_outputs_and_maybe_allow_retries(
stub, request, metadata=metadata())
raise_on_failure(
response,
custom_message=
f"Image predict failed for the {title} model (ID: {model_id}).",
)
def test_predict_video_url(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
video=resources_pb2.Video(url=CONAN_GIF_VIDEO_URL)))
],
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.frames) > 0
for frame in response.outputs[0].data.frames:
assert len(frame.data.concepts) > 0
def test_predict_image_url_with_min_value(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL, ), ), )
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(min_value=0.98))),
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.concepts) > 0
for c in response.outputs[0].data.concepts:
assert c.value >= 0.98
def test_mixed_success_predict(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL))),
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=NON_EXISTING_IMAGE_URL))),
],
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
assert response.status.code == status_code_pb2.MIXED_STATUS
assert response.outputs[0].status.code == status_code_pb2.SUCCESS
assert response.outputs[
1].status.code == status_code_pb2.INPUT_DOWNLOAD_FAILED
def test_predict_image_bytes(channel):
stub = service_pb2_grpc.V2Stub(channel)
with open(RED_TRUCK_IMAGE_FILE_PATH, "rb") as f:
file_bytes = f.read()
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(base64=file_bytes)))
],
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.concepts) > 0
def test_predict_video_bytes(channel):
stub = service_pb2_grpc.V2Stub(channel)
with open(TOY_VIDEO_FILE_PATH, "rb") as f:
file_bytes = f.read()
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
video=resources_pb2.Video(base64=file_bytes)))
],
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.frames) > 0
for frame in response.outputs[0].data.frames:
assert len(frame.data.concepts) > 0
def test_predict_video_url_with_min_value(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
video=resources_pb2.Video(url=CONAN_GIF_VIDEO_URL)))
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(min_value=0.95))),
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.frames) > 0
for frame in response.outputs[0].data.frames:
assert len(frame.data.concepts) > 0
for concept in frame.data.concepts:
assert concept.value >= 0.95
def test_video_predict_on_public_models(channel):
stub = service_pb2_grpc.V2Stub(channel)
models_with_no_video_support = {
"color",
}
for title, model_id in MODEL_TITLE_AND_ID_PAIRS:
if title in models_with_no_video_support:
continue
request = service_pb2.PostModelOutputsRequest(
model_id=model_id,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
video=resources_pb2.Video(url=BEER_VIDEO_URL)))
],
)
response = post_model_outputs_and_maybe_allow_retries(
stub, request, metadata=metadata())
raise_on_failure(
response,
custom_message=
f"Video predict failed for the {title} model (ID: {model_id}).",
)
def test_deep_classification_training_with_queries():
stub = service_pb2_grpc.V2Stub(ClarifaiChannel.get_grpc_channel())
app_id = "my-app-" + uuid.uuid4().hex[:20]
post_apps_response = stub.PostApps(
service_pb2.PostAppsRequest(apps=[
resources_pb2.App(
id=app_id,
default_workflow_id="General",
)
]),
metadata=pat_key_metadata(),
)
raise_on_failure(post_apps_response)
post_keys_response = stub.PostKeys(
service_pb2.PostKeysRequest(keys=[
resources_pb2.Key(
description="All scopes",
scopes=["All"],
apps=[resources_pb2.App(id=app_id, user_id="me")],
)
], ),
metadata=pat_key_metadata(),
)
raise_on_failure(post_keys_response)
api_key = post_keys_response.keys[0].id
template_name = "classification_cifar10_v1"
model_id = "my-deep-classif-" + uuid.uuid4().hex[:15]
model_type = _get_model_type_for_template(stub, api_key, template_name)
train_info_params = struct_pb2.Struct()
train_info_params.update({
"template": template_name,
"num_epochs": 2,
})
post_models_response = stub.PostModels(
service_pb2.PostModelsRequest(models=[
resources_pb2.Model(
id=model_id,
model_type_id=model_type.id,
train_info=resources_pb2.TrainInfo(params=train_info_params),
output_info=resources_pb2.OutputInfo(data=resources_pb2.Data(
concepts=[
resources_pb2.Concept(id="train-concept"),
resources_pb2.Concept(id="test-only-concept"),
]), ),
)
]),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_models_response)
train_and_test = ["train", "test"]
inputs = []
annotations = []
for i, url in enumerate(URLS):
input_id = str(i)
inputs.append(
resources_pb2.Input(
id=input_id,
data=resources_pb2.Data(image=resources_pb2.Image(url=url))))
train_annotation_info = struct_pb2.Struct()
train_annotation_info.update({"split": train_and_test[i % 2]})
ann = resources_pb2.Annotation(
input_id=input_id,
annotation_info=train_annotation_info,
data=resources_pb2.Data(
concepts=[resources_pb2.Concept(id="train-concept", value=1)]),
)
# Add an extra concept to the test set which show should up in evals, but have a bad score since there is
# no instance of it in the train set.
if i % 2 == 1:
ann.data.concepts.append(
resources_pb2.Concept(id="test-only-concept", value=1))
annotations.append(ann)
post_inputs_response = stub.PostInputs(
service_pb2.PostInputsRequest(inputs=inputs),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_inputs_response)
wait_for_inputs_upload(stub, api_key_metadata(api_key),
[str(i) for i in range(len(URLS))])
post_annotations_response = stub.PostAnnotations(
service_pb2.PostAnnotationsRequest(annotations=annotations),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_annotations_response)
train_annotation_info = struct_pb2.Struct()
train_annotation_info.update({"split": "train"})
train_query = resources_pb2.Query(ands=[
resources_pb2.And(annotation=resources_pb2.Annotation(
annotation_info=train_annotation_info)),
])
test_annotation_info = struct_pb2.Struct()
test_annotation_info.update({"split": "train"})
test_query = resources_pb2.Query(ands=[
resources_pb2.And(
negate=True,
annotation=resources_pb2.Annotation(
annotation_info=test_annotation_info),
),
])
post_model_versions_response = stub.PostModelVersions(
service_pb2.PostModelVersionsRequest(
model_id=model_id,
train_search=resources_pb2.Search(query=train_query),
test_search=resources_pb2.Search(query=test_query),
),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_model_versions_response)
model_version_id = post_model_versions_response.model.model_version.id
wait_for_model_trained(stub, api_key_metadata(api_key), model_id,
model_version_id)
post_model_outputs_request = service_pb2.PostModelOutputsRequest(
model_id=model_id,
version_id=model_version_id,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=URLS[0])))
],
)
post_model_outputs_response = post_model_outputs_and_maybe_allow_retries(
stub, post_model_outputs_request, metadata=api_key_metadata(api_key))
raise_on_failure(post_model_outputs_response)
concepts = post_model_outputs_response.outputs[0].data.concepts
assert len(concepts) == 2
assert concepts[0].id == "train-concept"
assert concepts[1].id == "test-only-concept"
assert concepts[1].value <= 0.0001
delete_app_response = stub.DeleteApp(
service_pb2.DeleteAppRequest(user_app_id=resources_pb2.UserAppIDSet(
user_id="me", app_id=app_id)),
metadata=pat_key_metadata(),
)
raise_on_failure(delete_app_response)
