def test_predict_image_url_with_selected_concepts(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL, ), ), )
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(select_concepts=[
resources_pb2.Concept(name="dog"),
resources_pb2.Concept(name="cat"),
]))),
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
concepts = response.outputs[0].data.concepts
assert len(concepts) == 2
dog_concept = [c for c in concepts if c.name == "dog"][0]
cat_concept = [c for c in concepts if c.name == "cat"][0]
assert dog_concept.value > cat_concept.value
def test_post_model_with_hyper_params(channel):
stub = service_pb2_grpc.V2Stub(channel)
model_id = uuid.uuid4().hex[:30]
hyper_params = struct_pb2.Struct()
hyper_params.update({
"MAX_NITEMS": 1000000,
"MIN_NITEMS": 1000,
"N_EPOCHS": 5,
"custom_training_cfg": "custom_training_1layer",
"custom_training_cfg_args": {},
})
post_response = stub.PostModels(
service_pb2.PostModelsRequest(models=[
resources_pb2.Model(
id=model_id,
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id="some-initial-concept")
], ),
output_config=resources_pb2.OutputConfig(
hyper_params=hyper_params),
),
)
]),
metadata=metadata(),
)
raise_on_failure(post_response)
assert (post_response.model.output_info.output_config.
hyper_params["custom_training_cfg"] == "custom_training_1layer")
delete_response = stub.DeleteModel(
service_pb2.DeleteModelRequest(model_id=model_id), metadata=metadata())
raise_on_failure(delete_response)
def test_predict_video_url_with_custom_sample_ms(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
video=resources_pb2.Video(url=BEER_VIDEO_URL)))
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(sample_ms=2000))),
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
# The expected time per frame is the middle between the start and the end of the frame
# (in milliseconds).
expected_time = 1000
assert len(response.outputs[0].data.frames) > 0
for frame in response.outputs[0].data.frames:
assert frame.frame_info.time == expected_time
expected_time += 2000
def request_call_integration(user_url, user_lan):
request = service_pb2.PostModelOutputsRequest(
model_id='aaa03c23b3724a16a56b629203edc62c',
inputs=[
resources_pb2.Input(data=resources_pb2.Data(image=resources_pb2.Image(url=user_url)))
],
model=resources_pb2.Model(
output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(
language=user_lan
)
)
))
response = stub.PostModelOutputs(request, metadata=metadata)
if response.status.code != status_code_pb2.SUCCESS:
raise Exception("Request failed, status code: " + str(response.status.code))
request_data=[]
for concept in response.outputs[0].data.concepts:
request_data.append(concept.name)
return request_data
def test_predict_image_url_with_max_concepts(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL, ), ), )
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(max_concepts=3))),
)
response = stub.PostModelOutputs(request, metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.concepts) == 3
def test_predict_video_url_with_max_concepts(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
video=resources_pb2.Video(url=CONAN_GIF_VIDEO_URL)))
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(max_concepts=3))),
)
response = stub.PostModelOutputs(request, metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.frames) > 0
for frame in response.outputs[0].data.frames:
assert len(frame.data.concepts) == 3
def test_predict_image_url_with_min_value(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL, ), ), )
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(min_value=0.98))),
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.concepts) > 0
for c in response.outputs[0].data.concepts:
assert c.value >= 0.98
def test_predict_video_url_with_min_value(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
video=resources_pb2.Video(url=CONAN_GIF_VIDEO_URL)))
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(min_value=0.95))),
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
assert len(response.outputs[0].data.frames) > 0
for frame in response.outputs[0].data.frames:
assert len(frame.data.concepts) > 0
for concept in frame.data.concepts:
assert concept.value >= 0.95
def test_post_patch_get_train_evaluate_delete_model(channel):
stub = service_pb2_grpc.V2Stub(channel)
# Add some inputs with the concepts that we'll need in the model.
post_inputs_response = stub.PostInputs(
service_pb2.PostInputsRequest(
inputs=[
resources_pb2.Input(
data=resources_pb2.Data(
image=resources_pb2.Image(url=TRUCK_IMAGE_URL, allow_duplicate_url=True),
concepts=[resources_pb2.Concept(id="some-initial-concept")],
)
),
resources_pb2.Input(
data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL, allow_duplicate_url=True),
concepts=[resources_pb2.Concept(id="some-new-concept")],
)
),
]
),
metadata=metadata(),
)
raise_on_failure(post_inputs_response)
input_id_1 = post_inputs_response.inputs[0].id
input_id_2 = post_inputs_response.inputs[1].id
wait_for_inputs_upload(stub, metadata(), [input_id_1, input_id_2])
model_id = u"我的新模型-" + uuid.uuid4().hex
post_response = stub.PostModels(
service_pb2.PostModelsRequest(
models=[
resources_pb2.Model(
id=model_id,
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(
concepts=[resources_pb2.Concept(id="some-initial-concept")],
),
),
)
]
),
metadata=metadata(),
)
raise_on_failure(post_response)
try:
patch_response = stub.PatchModels(
service_pb2.PatchModelsRequest(
action="overwrite",
models=[
resources_pb2.Model(
id=model_id,
name="some new name",
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(
concepts=[resources_pb2.Concept(id="some-new-concept", value=1)]
),
),
)
],
),
metadata=metadata(),
)
raise_on_failure(patch_response)
get_response = stub.GetModelOutputInfo(
service_pb2.GetModelRequest(model_id=model_id), metadata=metadata()
)
raise_on_failure(get_response)
assert get_response.model.id == model_id
assert get_response.model.name == "some new name"
assert len(get_response.model.output_info.data.concepts) == 1
assert get_response.model.output_info.data.concepts[0].id == "some-new-concept"
post_model_versions_response = stub.PostModelVersions(
service_pb2.PostModelVersionsRequest(model_id=model_id), metadata=metadata()
)
raise_on_failure(post_model_versions_response)
model_version_id = post_model_versions_response.model.model_version.id
wait_for_model_trained(stub, metadata(), model_id, model_version_id)
post_model_version_metrics_response = stub.PostModelVersionMetrics(
service_pb2.PostModelVersionMetricsRequest(
model_id=model_id, version_id=model_version_id
),
metadata=metadata(),
)
raise_on_failure(post_model_version_metrics_response)
wait_for_model_evaluated(stub, metadata(), model_id, model_version_id)
finally:
delete_response = stub.DeleteModel(
service_pb2.DeleteModelRequest(model_id=model_id), metadata=metadata()
)
raise_on_failure(delete_response)
delete_inputs_response = stub.DeleteInputs(
service_pb2.DeleteInputsRequest(ids=[input_id_1, input_id_2]), metadata=metadata()
)
raise_on_failure(delete_inputs_response)
def test_model_creation_training_and_evaluation(channel):
model_id = str(uuid.uuid4())
stub = service_pb2_grpc.V2Stub(channel)
raise_on_failure(
stub.PostModels(
service_pb2.PostModelsRequest(
models=[
resources_pb2.Model(
id=model_id,
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(
concepts=[
resources_pb2.Concept(id="dog"),
resources_pb2.Concept(id="toddler"),
]
)
),
)
]
),
metadata=metadata(),
)
)
post_inputs_response = stub.PostInputs(
service_pb2.PostInputsRequest(
inputs=[
resources_pb2.Input(
data=resources_pb2.Data(
image=resources_pb2.Image(
url="https://samples.clarifai.com/dog2.jpeg",
allow_duplicate_url=True,
),
concepts=[resources_pb2.Concept(id="dog")],
)
),
resources_pb2.Input(
data=resources_pb2.Data(
image=resources_pb2.Image(
url="https://samples.clarifai.com/toddler-flowers.jpeg",
allow_duplicate_url=True,
),
concepts=[resources_pb2.Concept(id="toddler")],
)
),
]
),
metadata=metadata(),
)
raise_on_failure(post_inputs_response)
input_ids = [i.id for i in post_inputs_response.inputs]
wait_for_inputs_upload(stub, metadata, input_ids)
response = stub.PostModelVersions(
service_pb2.PostModelVersionsRequest(model_id=model_id), metadata=metadata()
)
raise_on_failure(response)
model_version_id = response.model.model_version.id
wait_for_model_trained(stub, metadata, model_id, model_version_id)
raise_on_failure(
stub.PostModelVersionMetrics(
service_pb2.PostModelVersionMetricsRequest(
model_id=model_id,
version_id=model_version_id,
),
metadata=metadata(),
)
)
wait_for_model_evaluated(stub, metadata, model_id, model_version_id)
response = stub.GetModelVersionMetrics(
service_pb2.GetModelVersionMetricsRequest(
model_id=model_id,
version_id=model_version_id,
fields=resources_pb2.FieldsValue(
confusion_matrix=True,
cooccurrence_matrix=True,
label_counts=True,
binary_metrics=True,
test_set=True,
),
),
metadata=metadata(),
)
raise_on_failure(response)
raise_on_failure(
stub.DeleteModel(service_pb2.DeleteModelRequest(model_id=model_id), metadata=metadata())
)
raise_on_failure(
stub.DeleteInputs(service_pb2.DeleteInputsRequest(ids=input_ids), metadata=metadata())
)
def test_deep_classification_training_with_queries():
stub = service_pb2_grpc.V2Stub(ClarifaiChannel.get_grpc_channel())
app_id = "my-app-" + uuid.uuid4().hex[:20]
post_apps_response = stub.PostApps(
service_pb2.PostAppsRequest(
apps=[
resources_pb2.App(
id=app_id,
default_workflow_id="General",
)
]
),
metadata=pat_key_metadata(),
)
raise_on_failure(post_apps_response)
post_keys_response = stub.PostKeys(
service_pb2.PostKeysRequest(
keys=[
resources_pb2.Key(
description="All scopes",
scopes=["All"],
apps=[resources_pb2.App(id=app_id, user_id="me")],
)
],
),
metadata=pat_key_metadata(),
)
raise_on_failure(post_keys_response)
api_key = post_keys_response.keys[0].id
template_name = "classification_cifar10_v1"
model_id = "my-deep-classification-" + uuid.uuid4().hex
model_type = _get_model_type_for_template(stub, api_key, template_name)
train_info_params = struct_pb2.Struct()
train_info_params.update(
{
"template": template_name,
"num_epochs": 2,
"num_gpus": 0,
}
)
post_models_response = stub.PostModels(
service_pb2.PostModelsRequest(
models=[
resources_pb2.Model(
id=model_id,
model_type_id=model_type.id,
train_info=resources_pb2.TrainInfo(params=train_info_params),
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(
concepts=[
resources_pb2.Concept(id="train-concept"),
resources_pb2.Concept(id="test-only-concept"),
]
),
),
)
]
),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_models_response)
train_and_test = ["train", "test"]
inputs = []
annotations = []
for i, url in enumerate(URLS):
input_id = str(i)
inputs.append(
resources_pb2.Input(
id=input_id, data=resources_pb2.Data(image=resources_pb2.Image(url=url))
)
)
train_annotation_info = struct_pb2.Struct()
train_annotation_info.update({"split": train_and_test[i % 2]})
ann = resources_pb2.Annotation(
input_id=input_id,
annotation_info=train_annotation_info,
data=resources_pb2.Data(concepts=[resources_pb2.Concept(id="train-concept", value=1)]),
)
# Add an extra concept to the test set which show should up in evals, but have a bad score since there is
# no instance of it in the train set.
if i % 2 == 1:
ann.data.concepts.append(resources_pb2.Concept(id="test-only-concept", value=1))
annotations.append(ann)
post_inputs_response = stub.PostInputs(
service_pb2.PostInputsRequest(inputs=inputs),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_inputs_response)
wait_for_inputs_upload(stub, api_key_metadata(api_key), [str(i) for i in range(len(URLS))])
post_annotations_response = stub.PostAnnotations(
service_pb2.PostAnnotationsRequest(annotations=annotations),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_annotations_response)
train_annotation_info = struct_pb2.Struct()
train_annotation_info.update({"split": "train"})
train_query = resources_pb2.Query(
ands=[
resources_pb2.And(
annotation=resources_pb2.Annotation(annotation_info=train_annotation_info)
),
]
)
test_annotation_info = struct_pb2.Struct()
test_annotation_info.update({"split": "train"})
test_query = resources_pb2.Query(
ands=[
resources_pb2.And(
negate=True,
annotation=resources_pb2.Annotation(annotation_info=test_annotation_info),
),
]
)
post_model_versions_response = stub.PostModelVersions(
service_pb2.PostModelVersionsRequest(
model_id=model_id,
train_search=resources_pb2.Search(query=train_query),
test_search=resources_pb2.Search(query=test_query),
),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_model_versions_response)
model_version_id = post_model_versions_response.model.model_version.id
wait_for_model_trained(stub, api_key_metadata(api_key), model_id, model_version_id)
post_model_outputs_response = stub.PostModelOutputs(
service_pb2.PostModelOutputsRequest(
model_id=model_id,
version_id=model_version_id,
inputs=[
resources_pb2.Input(
data=resources_pb2.Data(image=resources_pb2.Image(url=URLS[0]))
)
],
),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_model_outputs_response)
concepts = post_model_outputs_response.outputs[0].data.concepts
assert len(concepts) == 2
assert concepts[0].id == "train-concept"
assert concepts[1].id == "test-only-concept"
assert concepts[1].value <= 0.0001
delete_app_response = stub.DeleteApp(
service_pb2.DeleteAppRequest(
user_app_id=resources_pb2.UserAppIDSet(user_id="me", app_id=app_id)
),
metadata=pat_key_metadata(),
)
raise_on_failure(delete_app_response)
def test_post_patch_get_delete_workflow(channel):
stub = service_pb2_grpc.V2Stub(channel)
# Max workflow ID is capped at 32 chars.
workflow_id = "food-and-general-" + uuid.uuid4().hex[:15]
post_workflows_response = stub.PostWorkflows(
service_pb2.PostWorkflowsRequest(
workflows=[
resources_pb2.Workflow(
id=workflow_id,
nodes=[
resources_pb2.WorkflowNode(
id="the-general-node",
# This is the public General model.
model=resources_pb2.Model(
id="aaa03c23b3724a16a56b629203edc62c",
model_version=resources_pb2.ModelVersion(
id="aa7f35c01e0642fda5cf400f543e7c40"),
),
),
resources_pb2.WorkflowNode(
id="the-food-node",
# This is the public Food model.
model=resources_pb2.Model(
id="bd367be194cf45149e75f01d59f77ba7",
model_version=resources_pb2.ModelVersion(
id="dfebc169854e429086aceb8368662641"),
),
),
],
)
], ),
metadata=metadata(),
)
raise_on_failure(post_workflows_response)
try:
# Update the workflow to use an older General model version.
patch_workflows_response = stub.PatchWorkflows(
service_pb2.PatchWorkflowsRequest(
action="overwrite",
workflows=[
resources_pb2.Workflow(
id=workflow_id,
nodes=[
resources_pb2.WorkflowNode(
id="the-general-node",
# This is the public General model, but the version is from 2016.
model=resources_pb2.Model(
id="aaa03c23b3724a16a56b629203edc62c",
model_version=resources_pb2.ModelVersion(
id="aa9ca48295b37401f8af92ad1af0d91d"),
),
),
resources_pb2.WorkflowNode(
id="the-food-node",
# This is the public Food model.
model=resources_pb2.Model(
id="bd367be194cf45149e75f01d59f77ba7",
model_version=resources_pb2.ModelVersion(
id="dfebc169854e429086aceb8368662641"),
),
),
],
)
],
),
metadata=metadata(),
)
raise_on_failure(patch_workflows_response)
get_workflow_response = stub.GetWorkflow(
service_pb2.GetWorkflowRequest(workflow_id=workflow_id),
metadata=metadata())
raise_on_failure(get_workflow_response)
# Make sure that after patching the workflow actually has the older General model version
# ID.
assert (get_workflow_response.workflow.nodes[0].model.model_version.id
== "aa9ca48295b37401f8af92ad1af0d91d")
finally:
delete_workflow_response = stub.DeleteWorkflow(
service_pb2.DeleteWorkflowRequest(workflow_id=workflow_id),
metadata=metadata())
raise_on_failure(delete_workflow_response)