def test_model_creation_training_and_evaluation(channel): model_id = str(uuid.uuid4()) stub = service_pb2_grpc.V2Stub(channel) raise_on_failure( stub.PostModels( service_pb2.PostModelsRequest( models=[ resources_pb2.Model( id=model_id, output_info=resources_pb2.OutputInfo( data=resources_pb2.Data( concepts=[ resources_pb2.Concept(id="dog"), resources_pb2.Concept(id="toddler"), ] ) ), ) ] ), metadata=metadata(), ) ) post_inputs_response = stub.PostInputs( service_pb2.PostInputsRequest( inputs=[ resources_pb2.Input( data=resources_pb2.Data( image=resources_pb2.Image( url="https://samples.clarifai.com/dog2.jpeg", allow_duplicate_url=True, ), concepts=[resources_pb2.Concept(id="dog")], ) ), resources_pb2.Input( data=resources_pb2.Data( image=resources_pb2.Image( url="https://samples.clarifai.com/toddler-flowers.jpeg", allow_duplicate_url=True, ), concepts=[resources_pb2.Concept(id="toddler")], ) ), ] ), metadata=metadata(), ) raise_on_failure(post_inputs_response) input_ids = [i.id for i in post_inputs_response.inputs] wait_for_inputs_upload(stub, metadata, input_ids) response = stub.PostModelVersions( service_pb2.PostModelVersionsRequest(model_id=model_id), metadata=metadata() ) raise_on_failure(response) model_version_id = response.model.model_version.id wait_for_model_trained(stub, metadata, model_id, model_version_id) raise_on_failure( stub.PostModelVersionMetrics( service_pb2.PostModelVersionMetricsRequest( model_id=model_id, version_id=model_version_id, ), metadata=metadata(), ) ) wait_for_model_evaluated(stub, metadata, model_id, model_version_id) response = stub.GetModelVersionMetrics( service_pb2.GetModelVersionMetricsRequest( model_id=model_id, version_id=model_version_id, fields=resources_pb2.FieldsValue( confusion_matrix=True, cooccurrence_matrix=True, label_counts=True, binary_metrics=True, test_set=True, ), ), metadata=metadata(), ) raise_on_failure(response) raise_on_failure( stub.DeleteModel(service_pb2.DeleteModelRequest(model_id=model_id), metadata=metadata()) ) raise_on_failure( stub.DeleteInputs(service_pb2.DeleteInputsRequest(ids=input_ids), metadata=metadata()) )
def test_post_patch_get_train_evaluate_delete_model(channel): stub = service_pb2_grpc.V2Stub(channel) # Add some inputs with the concepts that we'll need in the model. post_inputs_response = stub.PostInputs( service_pb2.PostInputsRequest( inputs=[ resources_pb2.Input( data=resources_pb2.Data( image=resources_pb2.Image(url=TRUCK_IMAGE_URL, allow_duplicate_url=True), concepts=[resources_pb2.Concept(id="some-initial-concept")], ) ), resources_pb2.Input( data=resources_pb2.Data( image=resources_pb2.Image(url=DOG_IMAGE_URL, allow_duplicate_url=True), concepts=[resources_pb2.Concept(id="some-new-concept")], ) ), ] ), metadata=metadata(), ) raise_on_failure(post_inputs_response) input_id_1 = post_inputs_response.inputs[0].id input_id_2 = post_inputs_response.inputs[1].id wait_for_inputs_upload(stub, metadata(), [input_id_1, input_id_2]) model_id = u"我的新模型-" + uuid.uuid4().hex post_response = stub.PostModels( service_pb2.PostModelsRequest( models=[ resources_pb2.Model( id=model_id, output_info=resources_pb2.OutputInfo( data=resources_pb2.Data( concepts=[resources_pb2.Concept(id="some-initial-concept")], ), ), ) ] ), metadata=metadata(), ) raise_on_failure(post_response) try: patch_response = stub.PatchModels( service_pb2.PatchModelsRequest( action="overwrite", models=[ resources_pb2.Model( id=model_id, name="some new name", output_info=resources_pb2.OutputInfo( data=resources_pb2.Data( concepts=[resources_pb2.Concept(id="some-new-concept", value=1)] ), ), ) ], ), metadata=metadata(), ) raise_on_failure(patch_response) get_response = stub.GetModelOutputInfo( service_pb2.GetModelRequest(model_id=model_id), metadata=metadata() ) raise_on_failure(get_response) assert get_response.model.id == model_id assert get_response.model.name == "some new name" assert len(get_response.model.output_info.data.concepts) == 1 assert get_response.model.output_info.data.concepts[0].id == "some-new-concept" post_model_versions_response = stub.PostModelVersions( service_pb2.PostModelVersionsRequest(model_id=model_id), metadata=metadata() ) raise_on_failure(post_model_versions_response) model_version_id = post_model_versions_response.model.model_version.id wait_for_model_trained(stub, metadata(), model_id, model_version_id) post_model_version_metrics_response = stub.PostModelVersionMetrics( service_pb2.PostModelVersionMetricsRequest( model_id=model_id, version_id=model_version_id ), metadata=metadata(), ) raise_on_failure(post_model_version_metrics_response) wait_for_model_evaluated(stub, metadata(), model_id, model_version_id) finally: delete_response = stub.DeleteModel( service_pb2.DeleteModelRequest(model_id=model_id), metadata=metadata() ) raise_on_failure(delete_response) delete_inputs_response = stub.DeleteInputs( service_pb2.DeleteInputsRequest(ids=[input_id_1, input_id_2]), metadata=metadata() ) raise_on_failure(delete_inputs_response)
def test_deep_classification_training_with_queries(): stub = service_pb2_grpc.V2Stub(ClarifaiChannel.get_grpc_channel()) app_id = "my-app-" + uuid.uuid4().hex[:20] post_apps_response = stub.PostApps( service_pb2.PostAppsRequest( apps=[ resources_pb2.App( id=app_id, default_workflow_id="General", ) ] ), metadata=pat_key_metadata(), ) raise_on_failure(post_apps_response) post_keys_response = stub.PostKeys( service_pb2.PostKeysRequest( keys=[ resources_pb2.Key( description="All scopes", scopes=["All"], apps=[resources_pb2.App(id=app_id, user_id="me")], ) ], ), metadata=pat_key_metadata(), ) raise_on_failure(post_keys_response) api_key = post_keys_response.keys[0].id template_name = "classification_cifar10_v1" model_id = "my-deep-classification-" + uuid.uuid4().hex model_type = _get_model_type_for_template(stub, api_key, template_name) train_info_params = struct_pb2.Struct() train_info_params.update( { "template": template_name, "num_epochs": 2, "num_gpus": 0, } ) post_models_response = stub.PostModels( service_pb2.PostModelsRequest( models=[ resources_pb2.Model( id=model_id, model_type_id=model_type.id, train_info=resources_pb2.TrainInfo(params=train_info_params), output_info=resources_pb2.OutputInfo( data=resources_pb2.Data( concepts=[ resources_pb2.Concept(id="train-concept"), resources_pb2.Concept(id="test-only-concept"), ] ), ), ) ] ), metadata=api_key_metadata(api_key), ) raise_on_failure(post_models_response) train_and_test = ["train", "test"] inputs = [] annotations = [] for i, url in enumerate(URLS): input_id = str(i) inputs.append( resources_pb2.Input( id=input_id, data=resources_pb2.Data(image=resources_pb2.Image(url=url)) ) ) train_annotation_info = struct_pb2.Struct() train_annotation_info.update({"split": train_and_test[i % 2]}) ann = resources_pb2.Annotation( input_id=input_id, annotation_info=train_annotation_info, data=resources_pb2.Data(concepts=[resources_pb2.Concept(id="train-concept", value=1)]), ) # Add an extra concept to the test set which show should up in evals, but have a bad score since there is # no instance of it in the train set. if i % 2 == 1: ann.data.concepts.append(resources_pb2.Concept(id="test-only-concept", value=1)) annotations.append(ann) post_inputs_response = stub.PostInputs( service_pb2.PostInputsRequest(inputs=inputs), metadata=api_key_metadata(api_key), ) raise_on_failure(post_inputs_response) wait_for_inputs_upload(stub, api_key_metadata(api_key), [str(i) for i in range(len(URLS))]) post_annotations_response = stub.PostAnnotations( service_pb2.PostAnnotationsRequest(annotations=annotations), metadata=api_key_metadata(api_key), ) raise_on_failure(post_annotations_response) train_annotation_info = struct_pb2.Struct() train_annotation_info.update({"split": "train"}) train_query = resources_pb2.Query( ands=[ resources_pb2.And( annotation=resources_pb2.Annotation(annotation_info=train_annotation_info) ), ] ) test_annotation_info = struct_pb2.Struct() test_annotation_info.update({"split": "train"}) test_query = resources_pb2.Query( ands=[ resources_pb2.And( negate=True, annotation=resources_pb2.Annotation(annotation_info=test_annotation_info), ), ] ) post_model_versions_response = stub.PostModelVersions( service_pb2.PostModelVersionsRequest( model_id=model_id, train_search=resources_pb2.Search(query=train_query), test_search=resources_pb2.Search(query=test_query), ), metadata=api_key_metadata(api_key), ) raise_on_failure(post_model_versions_response) model_version_id = post_model_versions_response.model.model_version.id wait_for_model_trained(stub, api_key_metadata(api_key), model_id, model_version_id) post_model_outputs_response = stub.PostModelOutputs( service_pb2.PostModelOutputsRequest( model_id=model_id, version_id=model_version_id, inputs=[ resources_pb2.Input( data=resources_pb2.Data(image=resources_pb2.Image(url=URLS[0])) ) ], ), metadata=api_key_metadata(api_key), ) raise_on_failure(post_model_outputs_response) concepts = post_model_outputs_response.outputs[0].data.concepts assert len(concepts) == 2 assert concepts[0].id == "train-concept" assert concepts[1].id == "test-only-concept" assert concepts[1].value <= 0.0001 delete_app_response = stub.DeleteApp( service_pb2.DeleteAppRequest( user_app_id=resources_pb2.UserAppIDSet(user_id="me", app_id=app_id) ), metadata=pat_key_metadata(), ) raise_on_failure(delete_app_response)