def test_search_by_geo_box_and_annotated_name_and_predicted_name(channel):
stub = service_pb2_grpc.V2Stub(channel)
with SetupImage(stub) as input_:
my_concept_name = input_.data.concepts[0].name
response = stub.PostSearches(
service_pb2.PostSearchesRequest(
query=resources_pb2.Query(ands=[
resources_pb2.And(input=resources_pb2.Input(
data=resources_pb2.Data(geo=resources_pb2.Geo(geo_box=[
resources_pb2.GeoBoxedPoint(
geo_point=resources_pb2.GeoPoint(longitude=43,
latitude=54)),
resources_pb2.GeoBoxedPoint(
geo_point=resources_pb2.GeoPoint(longitude=45,
latitude=56)),
])))),
resources_pb2.And(input=resources_pb2.Input(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(name=my_concept_name,
value=1)
]))),
resources_pb2.And(output=resources_pb2.Output(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(name="dog", value=1)
]))),
]),
pagination=service_pb2.Pagination(page=1, per_page=1000),
),
metadata=metadata(),
)
raise_on_failure(response)
assert len(response.hits) > 0
assert input_.id in [hit.input.id for hit in response.hits]
def test_post_list_patch_get_delete_image(channel):
stub = service_pb2_grpc.V2Stub(channel)
post_response = stub.PostInputs(
service_pb2.PostInputsRequest(
inputs=[
resources_pb2.Input(
data=resources_pb2.Data(
image=resources_pb2.Image(url=TRUCK_IMAGE_URL, allow_duplicate_url=True),
concepts=[resources_pb2.Concept(id="some-concept")],
)
)
]
),
metadata=metadata(),
)
raise_on_failure(post_response)
input_id = post_response.inputs[0].id
try:
wait_for_inputs_upload(stub, metadata(), [input_id])
list_response = stub.ListInputs(
service_pb2.ListInputsRequest(per_page=1), metadata=metadata()
)
raise_on_failure(list_response)
assert len(list_response.inputs) == 1
# Most likely we don"t have that many inputs, so this should return 0.
list_response2 = stub.ListInputs(
service_pb2.ListInputsRequest(per_page=500, page=1000), metadata=metadata()
)
raise_on_failure(list_response2)
assert len(list_response2.inputs) == 0
patch_response = stub.PatchInputs(
service_pb2.PatchInputsRequest(
action="overwrite",
inputs=[
resources_pb2.Input(
id=input_id,
data=resources_pb2.Data(
concepts=[resources_pb2.Concept(id="some-new-concept")]
),
)
],
),
metadata=metadata(),
)
raise_on_failure(patch_response)
get_response = stub.GetInput(
service_pb2.GetInputRequest(input_id=input_id), metadata=metadata()
)
raise_on_failure(get_response)
assert get_response.input.data.concepts[0].name == "some-new-concept"
finally:
delete_request = service_pb2.DeleteInputRequest(input_id=input_id)
delete_response = stub.DeleteInput(delete_request, metadata=metadata())
raise_on_failure(delete_response)
def test_predict_image_url_with_selected_concepts(channel):
stub = service_pb2_grpc.V2Stub(channel)
request = service_pb2.PostModelOutputsRequest(
model_id=GENERAL_MODEL_ID,
inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL, ), ), )
],
model=resources_pb2.Model(output_info=resources_pb2.OutputInfo(
output_config=resources_pb2.OutputConfig(select_concepts=[
resources_pb2.Concept(name="dog"),
resources_pb2.Concept(name="cat"),
]))),
)
response = post_model_outputs_and_maybe_allow_retries(stub,
request,
metadata=metadata())
raise_on_failure(response)
concepts = response.outputs[0].data.concepts
assert len(concepts) == 2
dog_concept = [c for c in concepts if c.name == "dog"][0]
cat_concept = [c for c in concepts if c.name == "cat"][0]
assert dog_concept.value > cat_concept.value
def test_concept_post_get_patch(channel):
stub = service_pb2_grpc.V2Stub(channel)
random_string = uuid.uuid4().hex
random_concept_id = "some-concept-id-狗-" + random_string
random_concept_name = "some-concept-name-的な-" + random_string
post_concepts_response = stub.PostConcepts(
service_pb2.PostConceptsRequest(concepts=[
resources_pb2.Concept(id=random_concept_id,
name=random_concept_name)
]),
metadata=metadata(),
)
raise_on_failure(post_concepts_response)
get_concepts_response = stub.GetConcept(
service_pb2.GetConceptRequest(concept_id=random_concept_id),
metadata=metadata())
raise_on_failure(get_concepts_response)
assert get_concepts_response.concept.id == random_concept_id
assert get_concepts_response.concept.name == random_concept_name
duplicated_post_concepts_response = stub.PostConcepts(
service_pb2.PostConceptsRequest(
concepts=[resources_pb2.Concept(id=random_concept_id, )]),
metadata=metadata(),
)
assert (duplicated_post_concepts_response.status.code ==
status_code_pb2.StatusCode.CONCEPTS_INVALID_REQUEST)
assert duplicated_post_concepts_response.status.description == "Invalid request"
assert "duplicate" in duplicated_post_concepts_response.status.details.lower(
)
post_concepts_searches_response = stub.PostConceptsSearches(
service_pb2.PostConceptsSearchesRequest(
concept_query=resources_pb2.ConceptQuery(
name=random_concept_name)),
metadata=metadata(),
)
raise_on_failure(post_concepts_searches_response)
assert random_concept_name in post_concepts_searches_response.concepts[
0].name
patch_concepts_response = stub.PatchConcepts(
service_pb2.PatchConceptsRequest(
action="overwrite",
concepts=[
resources_pb2.Concept(id=random_concept_id,
name="some new concept name")
],
),
metadata=metadata(),
)
raise_on_failure(patch_concepts_response)
def test_save_and_execute_search_by_id(channel):
stub = service_pb2_grpc.V2Stub(channel)
search_id = "my-search-id-" + uuid.uuid4().hex[:15]
with SetupImage(stub) as input_:
my_concept_id = input_.data.concepts[0].id
# This saves the search under an ID, but does not execute it / return any results.
save_search_response = stub.PostSearches(
service_pb2.PostSearchesRequest(searches=[
resources_pb2.Search(
id=search_id,
save=True,
query=resources_pb2.Query(ands=[
resources_pb2.And(input=resources_pb2.Input(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=my_concept_id,
value=1)
])))
]),
)
]),
metadata=metadata(),
)
raise_on_failure(save_search_response)
# Executing the search returns results.
post_search_by_id_response = stub.PostSearchesByID(
service_pb2.PostSearchesByIDRequest(id=search_id),
metadata=metadata(),
)
raise_on_failure(post_search_by_id_response)
assert len(post_search_by_id_response.hits) == 1
assert post_search_by_id_response.hits[0].input.id == input_.id
def test_post_model_with_hyper_params(channel):
stub = service_pb2_grpc.V2Stub(channel)
model_id = uuid.uuid4().hex[:30]
hyper_params = struct_pb2.Struct()
hyper_params.update({
"MAX_NITEMS": 1000000,
"MIN_NITEMS": 1000,
"N_EPOCHS": 5,
"custom_training_cfg": "custom_training_1layer",
"custom_training_cfg_args": {},
})
post_response = stub.PostModels(
service_pb2.PostModelsRequest(models=[
resources_pb2.Model(
id=model_id,
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id="some-initial-concept")
], ),
output_config=resources_pb2.OutputConfig(
hyper_params=hyper_params),
),
)
]),
metadata=metadata(),
)
raise_on_failure(post_response)
assert (post_response.model.output_info.output_config.
hyper_params["custom_training_cfg"] == "custom_training_1layer")
delete_response = stub.DeleteModel(
service_pb2.DeleteModelRequest(model_id=model_id), metadata=metadata())
raise_on_failure(delete_response)
def __enter__(self) -> resources_pb2.Input:
my_concept_id = "my-concept-id-" + uuid.uuid4().hex
my_concept_name = "my concept name " + uuid.uuid4().hex
image_metadata = struct_pb2.Struct()
image_metadata.update({
"some-key": "some-value",
"another-key": {
"inner-key": "inner-value"
}
})
post_response = self._stub.PostInputs(
service_pb2.PostInputsRequest(inputs=[
resources_pb2.Input(data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL,
allow_duplicate_url=True),
concepts=[
resources_pb2.Concept(
id=my_concept_id, name=my_concept_name, value=1)
],
metadata=image_metadata,
geo=resources_pb2.Geo(geo_point=resources_pb2.GeoPoint(
longitude=44, latitude=55)),
), )
]),
metadata=metadata(),
)
raise_on_failure(post_response)
self._input = post_response.inputs[0]
wait_for_inputs_upload(self._stub, metadata(), [self._input.id])
return self._input
def test_search_by_annotated_concept_name(channel):
stub = service_pb2_grpc.V2Stub(channel)
with SetupImage(stub) as input_:
my_concept_name = input_.data.concepts[0].name
response = stub.PostSearches(
service_pb2.PostSearchesRequest(query=resources_pb2.Query(ands=[
resources_pb2.And(input=resources_pb2.Input(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(name=my_concept_name, value=1)
])))
])),
metadata=metadata(),
)
raise_on_failure(response)
assert len(response.hits) == 1
assert response.hits[0].input.id == input_.id
def test_patching_public_concept_fails(channel):
stub = service_pb2_grpc.V2Stub(channel)
patch_concepts_searches_response = stub.PatchConcepts(
service_pb2.PatchConceptsRequest(
action="overwrite",
concepts=[
resources_pb2.Concept(
id="ai_98Xb0K3q", # The ID of a public concept.
name="this new name won't be applied",
)
],
),
metadata=metadata(),
)
assert (patch_concepts_searches_response.status.code ==
status_code_pb2.StatusCode.CONN_DOES_NOT_EXIST)
assert patch_concepts_searches_response.status.description == "Resource does not exist"
def test_search_by_predicted_concept_name(channel):
stub = service_pb2_grpc.V2Stub(channel)
with SetupImage(stub) as input_:
response = stub.PostSearches(
service_pb2.PostSearchesRequest(
query=resources_pb2.Query(ands=[
resources_pb2.And(output=resources_pb2.Output(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(name="dog", value=1)
])))
]),
pagination=service_pb2.Pagination(page=1, per_page=1000),
),
metadata=metadata(),
)
raise_on_failure(response)
assert len(response.hits) > 0
assert input_.id in [hit.input.id for hit in response.hits]
def test_post_annotations_searches(channel):
stub = service_pb2_grpc.V2Stub(channel)
palm_search_response = stub.PostConceptsSearches(
service_pb2.PostConceptsSearchesRequest(
concept_query=resources_pb2.ConceptQuery(name="palm")),
metadata=metadata(),
)
raise_on_failure(palm_search_response)
palm_concept_id = palm_search_response.concepts[0].id
water_search_response = stub.PostConceptsSearches(
service_pb2.PostConceptsSearchesRequest(
concept_query=resources_pb2.ConceptQuery(name="water")),
metadata=metadata(),
)
raise_on_failure(water_search_response)
water_concept_id = water_search_response.concepts[0].id
with SetupImage(stub) as input_:
post_palm_annotations_response = stub.PostAnnotations(
service_pb2.PostAnnotationsRequest(annotations=[
resources_pb2.Annotation(
input_id=input_.id,
data=resources_pb2.Data(regions=[
resources_pb2.Region(
region_info=resources_pb2.RegionInfo(
bounding_box=resources_pb2.BoundingBox(
top_row=0,
left_col=0,
bottom_row=0.45,
right_col=1)),
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=palm_concept_id,
value=1)
]),
),
]),
),
]),
metadata=metadata(),
)
raise_on_failure(post_palm_annotations_response)
post_water_annotations_response = stub.PostAnnotations(
service_pb2.PostAnnotationsRequest(annotations=[
resources_pb2.Annotation(
input_id=input_.id,
data=resources_pb2.Data(regions=[
resources_pb2.Region(
region_info=resources_pb2.RegionInfo(
bounding_box=resources_pb2.BoundingBox(
top_row=0.6,
left_col=0,
bottom_row=1,
right_col=0.98)),
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=water_concept_id,
value=1)
]),
),
]),
),
]),
metadata=metadata(),
)
raise_on_failure(post_water_annotations_response)
post_palm_annotations_searches_response = stub.PostAnnotationsSearches(
service_pb2.PostAnnotationsSearchesRequest(
searches=[
Search(query=resources_pb2.Query(filters=[
resources_pb2.Filter(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=palm_concept_id,
value=1)
])))
]))
],
pagination=service_pb2.Pagination(page=1, per_page=1000),
),
metadata=metadata(),
)
raise_on_failure(post_palm_annotations_searches_response)
assert input_.id in [
hit.input.id
for hit in post_palm_annotations_searches_response.hits
]
post_water_annotations_searches_response = stub.PostAnnotationsSearches(
service_pb2.PostAnnotationsSearchesRequest(
searches=[
Search(query=resources_pb2.Query(filters=[
resources_pb2.Filter(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=water_concept_id,
value=1)
])))
]))
],
pagination=service_pb2.Pagination(page=1, per_page=1000),
),
metadata=metadata(),
)
raise_on_failure(post_water_annotations_searches_response)
assert input_.id in [
hit.input.id
for hit in post_water_annotations_searches_response.hits
]
post_palm_and_water_annotations_searches_response = stub.PostAnnotationsSearches(
service_pb2.PostAnnotationsSearchesRequest(
searches=[
Search(query=resources_pb2.Query(filters=[
resources_pb2.Filter(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=palm_concept_id,
value=1)
])), ),
resources_pb2.Filter(
annotation=resources_pb2.Annotation(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=water_concept_id,
value=1)
]))),
]))
],
pagination=service_pb2.Pagination(page=1, per_page=1000),
),
metadata=metadata(),
)
raise_on_failure(post_palm_and_water_annotations_searches_response)
# No single annotation can have two concepts, so this will return false.
assert len(post_palm_and_water_annotations_searches_response.hits) == 0
def test_save_and_execute_annotations_search_by_id(channel):
stub = service_pb2_grpc.V2Stub(channel)
my_search_id = "my-search-id-" + uuid.uuid4().hex[:15]
my_concept_id = "my-anno-conc-" + uuid.uuid4().hex[:15]
with SetupImage(stub) as input1, SetupImage(stub) as input2:
list_annotations_response = stub.ListAnnotations(
service_pb2.ListAnnotationsRequest(
input_ids=[input1.id, input2.id]),
metadata=metadata(),
)
raise_on_failure(list_annotations_response)
input_id_to_annotation_id = {
an.input_id: an.id
for an in list_annotations_response.annotations
}
patch_annotations_response = stub.PatchAnnotations(
service_pb2.PatchAnnotationsRequest(
action="merge",
annotations=[
resources_pb2.Annotation(
id=input_id_to_annotation_id[input1.id],
input_id=input1.id,
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=my_concept_id, value=1)
]),
),
resources_pb2.Annotation(
id=input_id_to_annotation_id[input2.id],
input_id=input2.id,
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=my_concept_id, value=1)
]),
),
],
),
metadata=metadata(),
)
raise_on_failure(patch_annotations_response)
as_of = timestamp_pb2.Timestamp()
as_of.FromSeconds(int(time.time() + 5))
save_search_response = stub.PostSearches(
service_pb2.PostSearchesRequest(searches=[
resources_pb2.Search(
id=my_search_id,
save=True,
as_of=as_of,
query=resources_pb2.Query(ands=[
resources_pb2.And(input=resources_pb2.Input(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=my_concept_id,
value=1)
])))
]),
)
]),
metadata=metadata(),
)
raise_on_failure(save_search_response)
# Executing the search returns results.
post_search_by_id_response = stub.PostSearchesByID(
service_pb2.PostSearchesByIDRequest(id=my_search_id),
metadata=metadata(),
)
raise_on_failure(post_search_by_id_response)
hits = post_search_by_id_response.hits
assert len(hits) == 2
assert input1.id in [hit.input.id for hit in hits]
assert input2.id in [hit.input.id for hit in hits]
assert all(hit.score == 1 for hit in hits)
def test_post_patch_get_image_with_id_concepts_geo_and_metadata(channel):
stub = service_pb2_grpc.V2Stub(channel)
input_id = uuid.uuid4().hex
input_metadata = Struct()
input_metadata.update({
"key1": 123,
"key2": {
"inner-key1": "inner-val1",
"inner-key2": "inner-val2",
},
})
post_response = stub.PostInputs(
service_pb2.PostInputsRequest(inputs=[
resources_pb2.Input(
id=input_id,
data=resources_pb2.Data(
image=resources_pb2.Image(url=TRUCK_IMAGE_URL,
allow_duplicate_url=True),
concepts=[
resources_pb2.Concept(id="some-positive-concept",
value=1),
resources_pb2.Concept(id="some-negative-concept",
value=0),
],
geo=resources_pb2.Geo(geo_point=resources_pb2.GeoPoint(
longitude=55.0, latitude=66), ),
metadata=input_metadata,
),
)
]),
metadata=metadata(),
)
raise_on_failure(post_response)
wait_for_inputs_upload(stub, metadata(), [input_id])
get_response = stub.GetInput(
service_pb2.GetInputRequest(input_id=input_id), metadata=metadata())
raise_on_failure(get_response)
inp = get_response.input
assert inp.id == input_id
assert inp.data.concepts[0].id == "some-positive-concept"
assert inp.data.concepts[0].value == 1.0
assert inp.data.concepts[1].id == "some-negative-concept"
assert inp.data.concepts[1].value == 0.0
assert inp.data.metadata["key1"] == 123
assert inp.data.metadata["key2"]["inner-key1"] == "inner-val1"
assert inp.data.metadata["key2"]["inner-key2"] == "inner-val2"
assert inp.data.geo.geo_point.longitude == 55.0
assert inp.data.geo.geo_point.latitude == 66.0
new_metadata = Struct()
new_metadata.update({"new-key": "new-value"})
patch_response = stub.PatchInputs(
service_pb2.PatchInputsRequest(
action="merge",
inputs=[
resources_pb2.Input(
id=input_id,
data=resources_pb2.Data(
concepts=[
resources_pb2.Concept(
id="another-positive-concept", value=1)
],
geo=resources_pb2.Geo(geo_point=resources_pb2.GeoPoint(
longitude=77.0, latitude=88.0)),
metadata=new_metadata,
),
)
],
),
metadata=metadata(),
)
raise_on_failure(patch_response)
inp = patch_response.inputs[0]
assert inp.data.concepts[2].id == "another-positive-concept"
assert inp.data.geo.geo_point.longitude == 77.0
assert inp.data.geo.geo_point.latitude == 88.0
assert inp.data.metadata["new-key"] == "new-value"
assert (inp.data.metadata["key1"] == 123
) # Since we use the merge action, the old values should remain
delete_response = stub.DeleteInputs(
service_pb2.DeleteInputsRequest(ids=[input_id]), metadata=metadata())
raise_on_failure(delete_response)
def test_deep_classification_training_with_queries():
stub = service_pb2_grpc.V2Stub(ClarifaiChannel.get_grpc_channel())
app_id = "my-app-" + uuid.uuid4().hex[:20]
post_apps_response = stub.PostApps(
service_pb2.PostAppsRequest(
apps=[
resources_pb2.App(
id=app_id,
default_workflow_id="General",
)
]
),
metadata=pat_key_metadata(),
)
raise_on_failure(post_apps_response)
post_keys_response = stub.PostKeys(
service_pb2.PostKeysRequest(
keys=[
resources_pb2.Key(
description="All scopes",
scopes=["All"],
apps=[resources_pb2.App(id=app_id, user_id="me")],
)
],
),
metadata=pat_key_metadata(),
)
raise_on_failure(post_keys_response)
api_key = post_keys_response.keys[0].id
template_name = "classification_cifar10_v1"
model_id = "my-deep-classification-" + uuid.uuid4().hex
model_type = _get_model_type_for_template(stub, api_key, template_name)
train_info_params = struct_pb2.Struct()
train_info_params.update(
{
"template": template_name,
"num_epochs": 2,
"num_gpus": 0,
}
)
post_models_response = stub.PostModels(
service_pb2.PostModelsRequest(
models=[
resources_pb2.Model(
id=model_id,
model_type_id=model_type.id,
train_info=resources_pb2.TrainInfo(params=train_info_params),
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(
concepts=[
resources_pb2.Concept(id="train-concept"),
resources_pb2.Concept(id="test-only-concept"),
]
),
),
)
]
),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_models_response)
train_and_test = ["train", "test"]
inputs = []
annotations = []
for i, url in enumerate(URLS):
input_id = str(i)
inputs.append(
resources_pb2.Input(
id=input_id, data=resources_pb2.Data(image=resources_pb2.Image(url=url))
)
)
train_annotation_info = struct_pb2.Struct()
train_annotation_info.update({"split": train_and_test[i % 2]})
ann = resources_pb2.Annotation(
input_id=input_id,
annotation_info=train_annotation_info,
data=resources_pb2.Data(concepts=[resources_pb2.Concept(id="train-concept", value=1)]),
)
# Add an extra concept to the test set which show should up in evals, but have a bad score since there is
# no instance of it in the train set.
if i % 2 == 1:
ann.data.concepts.append(resources_pb2.Concept(id="test-only-concept", value=1))
annotations.append(ann)
post_inputs_response = stub.PostInputs(
service_pb2.PostInputsRequest(inputs=inputs),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_inputs_response)
wait_for_inputs_upload(stub, api_key_metadata(api_key), [str(i) for i in range(len(URLS))])
post_annotations_response = stub.PostAnnotations(
service_pb2.PostAnnotationsRequest(annotations=annotations),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_annotations_response)
train_annotation_info = struct_pb2.Struct()
train_annotation_info.update({"split": "train"})
train_query = resources_pb2.Query(
ands=[
resources_pb2.And(
annotation=resources_pb2.Annotation(annotation_info=train_annotation_info)
),
]
)
test_annotation_info = struct_pb2.Struct()
test_annotation_info.update({"split": "train"})
test_query = resources_pb2.Query(
ands=[
resources_pb2.And(
negate=True,
annotation=resources_pb2.Annotation(annotation_info=test_annotation_info),
),
]
)
post_model_versions_response = stub.PostModelVersions(
service_pb2.PostModelVersionsRequest(
model_id=model_id,
train_search=resources_pb2.Search(query=train_query),
test_search=resources_pb2.Search(query=test_query),
),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_model_versions_response)
model_version_id = post_model_versions_response.model.model_version.id
wait_for_model_trained(stub, api_key_metadata(api_key), model_id, model_version_id)
post_model_outputs_response = stub.PostModelOutputs(
service_pb2.PostModelOutputsRequest(
model_id=model_id,
version_id=model_version_id,
inputs=[
resources_pb2.Input(
data=resources_pb2.Data(image=resources_pb2.Image(url=URLS[0]))
)
],
),
metadata=api_key_metadata(api_key),
)
raise_on_failure(post_model_outputs_response)
concepts = post_model_outputs_response.outputs[0].data.concepts
assert len(concepts) == 2
assert concepts[0].id == "train-concept"
assert concepts[1].id == "test-only-concept"
assert concepts[1].value <= 0.0001
delete_app_response = stub.DeleteApp(
service_pb2.DeleteAppRequest(
user_app_id=resources_pb2.UserAppIDSet(user_id="me", app_id=app_id)
),
metadata=pat_key_metadata(),
)
raise_on_failure(delete_app_response)
def test_model_creation_training_and_evaluation(channel):
model_id = str(uuid.uuid4())
stub = service_pb2_grpc.V2Stub(channel)
raise_on_failure(
stub.PostModels(
service_pb2.PostModelsRequest(
models=[
resources_pb2.Model(
id=model_id,
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(
concepts=[
resources_pb2.Concept(id="dog"),
resources_pb2.Concept(id="toddler"),
]
)
),
)
]
),
metadata=metadata(),
)
)
post_inputs_response = stub.PostInputs(
service_pb2.PostInputsRequest(
inputs=[
resources_pb2.Input(
data=resources_pb2.Data(
image=resources_pb2.Image(
url="https://samples.clarifai.com/dog2.jpeg",
allow_duplicate_url=True,
),
concepts=[resources_pb2.Concept(id="dog")],
)
),
resources_pb2.Input(
data=resources_pb2.Data(
image=resources_pb2.Image(
url="https://samples.clarifai.com/toddler-flowers.jpeg",
allow_duplicate_url=True,
),
concepts=[resources_pb2.Concept(id="toddler")],
)
),
]
),
metadata=metadata(),
)
raise_on_failure(post_inputs_response)
input_ids = [i.id for i in post_inputs_response.inputs]
wait_for_inputs_upload(stub, metadata, input_ids)
response = stub.PostModelVersions(
service_pb2.PostModelVersionsRequest(model_id=model_id), metadata=metadata()
)
raise_on_failure(response)
model_version_id = response.model.model_version.id
wait_for_model_trained(stub, metadata, model_id, model_version_id)
raise_on_failure(
stub.PostModelVersionMetrics(
service_pb2.PostModelVersionMetricsRequest(
model_id=model_id,
version_id=model_version_id,
),
metadata=metadata(),
)
)
wait_for_model_evaluated(stub, metadata, model_id, model_version_id)
response = stub.GetModelVersionMetrics(
service_pb2.GetModelVersionMetricsRequest(
model_id=model_id,
version_id=model_version_id,
fields=resources_pb2.FieldsValue(
confusion_matrix=True,
cooccurrence_matrix=True,
label_counts=True,
binary_metrics=True,
test_set=True,
),
),
metadata=metadata(),
)
raise_on_failure(response)
raise_on_failure(
stub.DeleteModel(service_pb2.DeleteModelRequest(model_id=model_id), metadata=metadata())
)
raise_on_failure(
stub.DeleteInputs(service_pb2.DeleteInputsRequest(ids=input_ids), metadata=metadata())
)
def test_post_patch_get_train_evaluate_delete_model(channel):
stub = service_pb2_grpc.V2Stub(channel)
# Add some inputs with the concepts that we'll need in the model.
post_inputs_response = stub.PostInputs(
service_pb2.PostInputsRequest(
inputs=[
resources_pb2.Input(
data=resources_pb2.Data(
image=resources_pb2.Image(url=TRUCK_IMAGE_URL, allow_duplicate_url=True),
concepts=[resources_pb2.Concept(id="some-initial-concept")],
)
),
resources_pb2.Input(
data=resources_pb2.Data(
image=resources_pb2.Image(url=DOG_IMAGE_URL, allow_duplicate_url=True),
concepts=[resources_pb2.Concept(id="some-new-concept")],
)
),
]
),
metadata=metadata(),
)
raise_on_failure(post_inputs_response)
input_id_1 = post_inputs_response.inputs[0].id
input_id_2 = post_inputs_response.inputs[1].id
wait_for_inputs_upload(stub, metadata(), [input_id_1, input_id_2])
model_id = u"我的新模型-" + uuid.uuid4().hex
post_response = stub.PostModels(
service_pb2.PostModelsRequest(
models=[
resources_pb2.Model(
id=model_id,
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(
concepts=[resources_pb2.Concept(id="some-initial-concept")],
),
),
)
]
),
metadata=metadata(),
)
raise_on_failure(post_response)
try:
patch_response = stub.PatchModels(
service_pb2.PatchModelsRequest(
action="overwrite",
models=[
resources_pb2.Model(
id=model_id,
name="some new name",
output_info=resources_pb2.OutputInfo(
data=resources_pb2.Data(
concepts=[resources_pb2.Concept(id="some-new-concept", value=1)]
),
),
)
],
),
metadata=metadata(),
)
raise_on_failure(patch_response)
get_response = stub.GetModelOutputInfo(
service_pb2.GetModelRequest(model_id=model_id), metadata=metadata()
)
raise_on_failure(get_response)
assert get_response.model.id == model_id
assert get_response.model.name == "some new name"
assert len(get_response.model.output_info.data.concepts) == 1
assert get_response.model.output_info.data.concepts[0].id == "some-new-concept"
post_model_versions_response = stub.PostModelVersions(
service_pb2.PostModelVersionsRequest(model_id=model_id), metadata=metadata()
)
raise_on_failure(post_model_versions_response)
model_version_id = post_model_versions_response.model.model_version.id
wait_for_model_trained(stub, metadata(), model_id, model_version_id)
post_model_version_metrics_response = stub.PostModelVersionMetrics(
service_pb2.PostModelVersionMetricsRequest(
model_id=model_id, version_id=model_version_id
),
metadata=metadata(),
)
raise_on_failure(post_model_version_metrics_response)
wait_for_model_evaluated(stub, metadata(), model_id, model_version_id)
finally:
delete_response = stub.DeleteModel(
service_pb2.DeleteModelRequest(model_id=model_id), metadata=metadata()
)
raise_on_failure(delete_response)
delete_inputs_response = stub.DeleteInputs(
service_pb2.DeleteInputsRequest(ids=[input_id_1, input_id_2]), metadata=metadata()
)
raise_on_failure(delete_inputs_response)
def send_image(img_location, meta_data, metadata):
"""
Sends an image and its metadata to the Clarifai App that was set up (and the key was provided). This is fairly
custom for this particular project, but could be adapted to work with other types of projects.
"""
# Opens the PNG image location passed in.
with open(img_location, "rb") as f:
file_bytes = f.read()
#
# The following set of code generating the metadata to be sent with the API call is somewhat complex. It is
# unfortunately using methods included by a common but not universally known library (Protobuf by Google - for
# data serialization) as well as dealing with overcomming a current bug in the Clarifai system (currently
# being fixed).
#
input_metadata = Struct()
input_metadata.update(meta_data)
concepts = [
resources_pb2.Concept(id=meta_data["general_cancer"], value=1.)
]
project = get_project(meta_data["tcga_id"])
# This following if statement simply formats the metadata in the way that is desired. While it looks complex,
# that is fundementally all that is happening. The `concepts.append(...)` function is adding each entry of
# metadata/
if project is not None:
primary_sites = project["primary_site"].split(";")
for primary_site in primary_sites:
primary_site = (
primary_site[:31]) if len(primary_site) > 31 else primary_site
concepts.append(
resources_pb2.Concept(id=primary_site.replace(" ", "_"),
value=1.))
project_names = project["project_name"].split(";")
for project_name in project_names:
project_name = (
project_name[:31]) if len(project_name) > 31 else project_name
concepts.append(
resources_pb2.Concept(id=project_name.replace(" ", "_"),
value=1.))
tcga_cancer_types = project["tcga_cancer_type"].split(";")
for tcga_cancer_type in tcga_cancer_types:
tcga_cancer_type = (
tcga_cancer_type[:31]
) if len(tcga_cancer_type) > 31 else tcga_cancer_type
concepts.append(
resources_pb2.Concept(id=tcga_cancer_type.replace(" ", "_"),
value=1.))
#
# Call to the Clarifai API.
# This looks complicated, but this is really almost all boilerplate. In reality this is just a gRPC call (another
# commonly used Google library for communication across ports). In order to understand what is going on here
# would require a deeper understanding of gRPC works. However, for these purposes, the image bypes just needs
# to be put where `base64=` is at and the concepts are included in the `conceps=` line (the concepts we generated
# earlier). We are also adding the raw input metadata we contained in the optional (first) `metadata=` line.
# Finally, the final `metadata=` line is actually the App access metadata (the API Key from Clarifai) to prove
# that we have the credentials to access the App.
#
post_inputs_response = stub.PostInputs(
service_pb2.PostInputsRequest(inputs=[
resources_pb2.Input(
data=resources_pb2.Data(image=resources_pb2.Image(
base64=file_bytes),
concepts=concepts,
metadata=input_metadata))
]),
metadata=metadata)
#
# Finally we just check for any type of error and clean up the SVS file to save space. The removal of the SVS
# file could be removed if you were going to use them for another purpose.
#
if post_inputs_response.status.code != status_code_pb2.SUCCESS:
print("-- Failed Response: {}".format(post_inputs_response))
raise Exception("Post inputs failed, status: " +
post_inputs_response.status.details)
else:
os.remove(img_location)
svs_img = os.path.splitext(img_location)[0] + '.svs'
os.remove(svs_img)
