def create_model_version( self, model_name, model_path, model_type=None, version_desc=None, current_stage=ModelVersionStage.GENERATED ) -> Optional[ModelVersionDetail]: """ Create a new model version from given source and metric in Model Center. :param model_name: Name of registered model. This is expected to be unique in the backend store. :param model_path: Source path where the AIFlow model is stored. :param model_type: (Optional) Type of AIFlow registered model option. :param version_desc: (Optional) Description of registered model version. :param current_stage: (Optional) Stage of registered model version. :return: A single object of :py:class:`ai_flow.model_center.entity.ModelVersionDetail` created in model repository. """ request = CreateModelVersionRequest( model_meta=ModelMetaParam(model_name=stringValue(model_name)), model_version=ModelVersionParam( model_path=stringValue(model_path), model_type=stringValue(model_type), version_desc=stringValue(version_desc), current_stage=current_stage)) response = self.model_center_stub.createModelVersion(request) return ModelVersionDetail.from_proto( _parse_response(response, ModelVersionMeta()))
def register_metric_summary( self, metric_name: Text, metric_key: Text, metric_value: Text, metric_timestamp: int, model_version: Optional[Text] = None, job_execution_id: Optional[Text] = None ) -> Tuple[int, Text, Optional[MetricSummary]]: """ Register metric summary in Metric Center. :param metric_name: Name of registered metric summary. :param metric_key: Key of registered metric summary. :param metric_value: Value of registered metric summary. :param metric_timestamp: Timestamp of registered metric summary. :param model_version: (Optional) Version of the model version associated with the registered metric summary. :param job_execution_id: (Optional) ID of the job execution associated with the registered metric summary. :return: A single :py:class:`ai_flow.meta.metric_meta.MetricSummary` object. """ request = MetricSummaryRequest(metric_summary=MetricSummaryProto( metric_name=stringValue(metric_name), metric_key=stringValue(metric_key), metric_value=stringValue(metric_value), metric_timestamp=int64Value(metric_timestamp), model_version=stringValue(model_version), job_execution_id=stringValue(job_execution_id))) response = self.metric_stub.registerMetricSummary(request) if 0 == response.return_code: metric_summary_proto = response.metric_summary metric_summary = proto_to_metric_summary(metric_summary_proto) return response.return_code, response.return_msg, metric_summary else: return response.return_code, response.return_msg, None
def register_artifact(self, name: Text, artifact_type: Text = None, description: Text = None, uri: Text = None, properties: Properties = None) -> ArtifactMeta: """ register an artifact in metadata store. :param name: the name of the artifact :param artifact_type: the type of the artifact :param description: the description of the artifact :param uri: the uri of the artifact :param properties: the properties of the artifact :return: A single :py:class:`ai_flow.meta.artifact_meta.py.ArtifactMeta` object. """ artifact_request = ArtifactProto( name=name, artifact_type=stringValue(artifact_type), description=stringValue(description), uri=stringValue(uri), properties=properties) request = metadata_service_pb2.RegisterArtifactRequest( artifact=artifact_request) response = self.metadata_store_stub.registerArtifact(request) return _unwrap_artifact_response(response)
def update_artifact( self, artifact_name: Text, artifact_type: Text = None, description: Text = None, uri: Text = None, properties: Properties = None) -> Optional[ArtifactMeta]: """ update artifact in metadata store. :param artifact_name: the name of the artifact :param artifact_type: the type of the artifact :param description: the description of the artifact :param uri: the batch uri of the artifact :param properties: the properties of the artifact :return: A single :py:class:`ai_flow.meta.artifact_meta.py.ArtifactMeta` object if update successfully. """ request = metadata_service_pb2.UpdateArtifactRequest( name=artifact_name, artifact_type=stringValue(artifact_type), description=stringValue(description), uri=stringValue(uri), properties=properties) response = self.metadata_store_stub.updateArtifact(request) return _unwrap_artifact_response(response)
def update_model_version( self, model_name, model_version, model_path=None, model_type=None, version_desc=None, current_stage=None) -> Optional[ModelVersionDetail]: """ Update metadata for ModelVersion entity and metadata associated with a model version in backend. Either ``model_path`` or ``model_type`` or ``version_desc`` should be non-None. Backend raises exception if a registered model with given name does not exist. :param model_name: Name of registered model. This is expected to be unique in the backend store. :param model_version: User-defined version of registered model. :param model_path: (Optional) Source path where the AIFlow model is stored. :param model_type: (Optional) Type of AIFlow registered model option. :param version_desc: (Optional) Description of registered model version. :param current_stage: (Optional) Current stage of registered model version. :return: A single updated :py:class:`ai_flow.model_center.entity.ModelVersionDetail` object. """ request = UpdateModelVersionRequest( model_meta=ModelMetaParam( model_name=stringValue(model_name), model_version=stringValue(model_version)), model_version=ModelVersionParam( model_path=stringValue(model_path), model_type=stringValue(model_type), version_desc=stringValue(version_desc), current_stage=current_stage)) response = self.model_center_stub.updateModelVersion(request) return ModelVersionDetail.from_proto( _parse_response(response, ModelVersionMeta()))
def to_meta_proto(self): return ModelVersionMeta(model_name=self.model_name, model_version=self.model_version, model_path=stringValue(self.model_path), model_type=stringValue(self.model_type), version_desc=stringValue(self.version_desc), version_status=ModelVersionStatus.from_string( self.version_status), current_stage=ModelVersionStage.from_string( self.current_stage.upper()))
def to_detail_proto(self): if self.latest_model_version is None: return ProtoModelDetail(registered_model=RegisteredModelMeta( model_name=self.model_name, model_desc=stringValue(self.model_desc)), latest_model_version=None) else: return ProtoModelDetail( registered_model=RegisteredModelMeta( model_name=self.model_name, model_desc=stringValue(self.model_desc)), latest_model_version=self.latest_model_version.to_meta_proto())
def model_version_store_to_proto(model_version_relation, model_center_detail) -> ModelVersionProto: if model_version_relation is not None and model_center_detail is not None: return ModelVersionProto( version=stringValue(model_version_relation.version), model_id=int64Value(model_version_relation.model_id), project_snapshot_id=int64Value(model_version_relation.project_snapshot_id), model_path=stringValue(model_center_detail.model_path), model_type=stringValue(model_center_detail.model_type), version_desc=stringValue(model_center_detail.version_desc), current_stage=ModelVersionStage.Value(model_center_detail.current_stage.upper())) else: return None
def artifact_meta_to_proto(artifact_meta: ArtifactMeta) -> ArtifactProto: if artifact_meta is None: return None else: return ArtifactProto( uuid=artifact_meta.uuid, name=artifact_meta.name, properties=artifact_meta.properties, artifact_type=stringValue(artifact_meta.artifact_type), description=stringValue(artifact_meta.description), uri=stringValue(artifact_meta.uri), create_time=int64Value(artifact_meta.create_time), update_time=int64Value(artifact_meta.update_time))
def dataset_meta_to_proto(dataset_mata) -> DatasetMeta: if dataset_mata is None: return None else: if dataset_mata.schema is not None: name_list = dataset_mata.schema.name_list type_list = dataset_mata.schema.type_list data_type_list = [] if type_list is not None: for data_type in type_list: data_type_list.append(DataTypeProto.Value(data_type)) else: data_type_list = None else: name_list = None data_type_list = None schema = SchemaProto(name_list=name_list, type_list=data_type_list) return DatasetProto( uuid=dataset_mata.uuid, name=dataset_mata.name, properties=dataset_mata.properties, data_format=stringValue(dataset_mata.data_format), description=stringValue(dataset_mata.description), uri=stringValue(dataset_mata.uri), create_time=int64Value(dataset_mata.create_time), update_time=int64Value(dataset_mata.update_time), schema=schema, catalog_name=stringValue(dataset_mata.catalog_name), catalog_type=stringValue(dataset_mata.catalog_type), catalog_database=stringValue(dataset_mata.catalog_database), catalog_connection_uri=stringValue(dataset_mata.catalog_connection_uri), catalog_table=stringValue(dataset_mata.catalog_table))
def delete_model_version(self, model_name, model_version) -> ModelVersion: """ Delete model version by model name and version in Model Center backend. :param model_name: Name of registered model. This is expected to be unique in the backend store. :param model_version: User-defined version of registered model. :return: A single :py:class:`ai_flow.entities.model_registry.ModelVersion` object. """ request = DeleteModelVersionRequest(model_meta=ModelMetaParam( model_name=stringValue(model_name), model_version=stringValue(model_version))) response = self.model_center_stub.deleteModelVersion(request) return ModelVersion.from_resp_proto( _parse_response(response, ModelMetaParam()))
def list_model_metric_metas( self, model_name: Text, project_name: Optional[Text] = None ) -> Tuple[int, Text, Union[None, MetricMeta, List[MetricMeta]]]: """ List of model metric metadata filter by model name and project name for Metric Center. :param model_name: Name of the model associated with the registered metric meta. :param project_name: Name of the project associated with the registered metric meta. :return: List of :py:class:`ai_flow.meta.metric_meta.MetricMeta` objects. """ request = ListModelMetricMetasRequest( model_name=model_name, project_name=stringValue(project_name)) response = self.metric_stub.listModelMetricMetas(request) if 0 == response.return_code: repeated_metric_meta_proto = response.metric_metas if 1 == len(repeated_metric_meta_proto): metric_meta = proto_to_metric_meta( repeated_metric_meta_proto[0]) return response.return_code, response.return_msg, metric_meta else: res = [] for metric_meta_proto in repeated_metric_meta_proto: res.append(proto_to_metric_meta(metric_meta_proto)) return response.return_code, response.return_msg, res else: return response.return_code, response.return_msg, None
def get_model_version_detail( self, model_name, model_version) -> Optional[ModelVersionDetail]: """ Get model version detail filter by model name and model version for Model Center. :param model_name: Name of registered model. This is expected to be unique in the backend store. :param model_version: User-defined version of registered model. :return: A single :py:class:`ai_flow.entities.model_registry.ModelVersionDetail` object. """ request = GetModelVersionDetailRequest(model_meta=ModelMetaParam( model_name=stringValue(model_name), model_version=stringValue(model_version))) response = self.model_center_stub.getModelVersionDetail(request) return ModelVersionDetail.from_proto( _parse_response(response, ModelVersionMeta()))
def model_meta_to_proto(model_relation, model_center_detail) -> ModelProto: if model_relation is not None and model_center_detail is not None: return ModelProto(uuid=model_relation.uuid, name=model_relation.name, project_id=int64Value(model_relation.project_id), model_desc=stringValue(model_center_detail.model_desc)) else: return None
def project_meta_to_proto(project_meta: ProjectMeta) -> ProjectProto: if project_meta is None: return None else: return ProjectProto( uuid=project_meta.uuid, name=project_meta.name, properties=project_meta.properties, uri=stringValue(project_meta.uri))
def model_version_relation_meta_to_proto( model_version_relation: ModelVersionRelationMeta) -> ModelVersionRelationProto: if model_version_relation is None: return None else: return ModelVersionRelationProto( version=stringValue(model_version_relation.version), model_id=int64Value(model_version_relation.model_id), project_snapshot_id=int64Value(model_version_relation.project_snapshot_id))
def register_metric_meta( self, metric_name: Text, metric_type: MetricType, project_name: Text, metric_desc: Optional[Text] = None, dataset_name: Optional[Text] = None, model_name: Optional[Text] = None, job_name: Optional[Text] = None, start_time: int = None, end_time: int = None, uri: Optional[Text] = None, tags: Optional[Text] = None, properties: Properties = None ) -> Tuple[int, Text, Optional[MetricMeta]]: """ Register metric metadata in Metric Center. :param metric_name: Name of registered metric meta. This is expected to be unique in the backend store. :param metric_type: Type of registered metric meta. :param project_name: Name of the project associated with the registered metric meta. :param metric_desc: (Optional) Description of registered metric meta. :param dataset_name: (Optional) Name of the dataset associated with the registered metric meta. :param model_name: (Optional) Name of the model associated with the registered metric meta. :param job_name: (Optional) Name of the job associated with the registered metric meta. :param start_time: (Optional) Start time of registered metric meta. :param end_time: (Optional) End time of registered metric meta. :param uri: (Optional) Uri of registered metric meta. :param tags: (Optional) Tags of registered metric meta. :param properties: (Optional) Properties of registered metric meta. :return: A single :py:class:`ai_flow.meta.metric_meta.MetricMeta` object. """ request = MetricMetaRequest(metric_meta=MetricMetaProto( metric_name=stringValue(metric_name), metric_type=MetricTypeProto.Value(metric_type.value), metric_desc=stringValue(metric_desc), project_name=stringValue(project_name), dataset_name=stringValue(dataset_name), model_name=stringValue(model_name), job_name=stringValue(job_name), start_time=int64Value(start_time), end_time=int64Value(end_time), uri=stringValue(uri), tags=stringValue(tags), properties=properties)) response = self.metric_stub.registerMetricMeta(request) if 0 == response.return_code: metric_meta_proto = response.metric_meta metric_meta = proto_to_metric_meta(metric_meta_proto) return response.return_code, response.return_msg, metric_meta else: return response.return_code, response.return_msg, None
def create_registered_model( self, model_name, model_desc=None) -> Optional[RegisteredModelDetail]: """ Create a new registered model from given type in Model Center. :param model_name: Name of registered model. This is expected to be unique in the backend store. :param model_desc: (Optional) Description of registered model. :return: A single object of :py:class:`ai_flow.model_center.entity.RegisteredModelDetail` created in Model Center. """ request = CreateRegisteredModelRequest( registered_model=RegisteredModelParam( model_name=stringValue(model_name), model_desc=stringValue(model_desc))) response = self.model_center_stub.createRegisteredModel(request) return RegisteredModelDetail.from_proto( _parse_response(response, RegisteredModelMeta()))
def list_metric_summaries( self, metric_name: Optional[Text] = None, metric_key: Optional[Text] = None, model_version: Optional[Text] = None, start_time: int = None, end_time=None ) -> Tuple[int, Text, Union[None, MetricSummary, List[MetricSummary]]]: """ List of metric summaries filter by metric summary fields for Metric Center. :param metric_name: (Optional) Name of filtered metric summary. :param metric_key: (Optional) Key of filtered metric summary. :param model_version: (Optional) Version of the model version associated with the registered metric summary. :param start_time: (Optional) Start time for timestamp filtered metric summary. :param end_time: (Optional) End time for timestamp filtered metric summary. :return: List of :py:class:`ai_flow.meta.metric_meta.MetricSummary` objects. """ request = ListMetricSummariesRequest( metric_name=stringValue(metric_name), metric_key=stringValue(metric_key), model_version=stringValue(model_version), start_time=int64Value(start_time), end_time=int64Value(end_time)) response = self.metric_stub.listMetricSummaries(request) if 0 == response.return_code: repeated_metric_summary_proto = response.metric_summaries if 1 == len(repeated_metric_summary_proto): metric_summary = proto_to_metric_summary( repeated_metric_summary_proto[0]) return response.return_code, response.return_msg, metric_summary else: res = [] for metric_summary_proto in repeated_metric_summary_proto: res.append(proto_to_metric_summary(metric_summary_proto)) return response.return_code, response.return_msg, res else: return response.return_code, response.return_msg, None
def delete_registered_model(self, model_name) -> RegisteredModelDetail: """ Delete registered model by model name in Model Center backend. :param model_name: Name of registered model. This is expected to be unique in the backend store. :return: A single :py:class:`ai_flow.entities.model_registry.RegisteredModel` object. """ request = DeleteRegisteredModelRequest(model_meta=ModelMetaParam( model_name=stringValue(model_name))) response = self.model_center_stub.deleteRegisteredModel(request) return RegisteredModel.from_resp_proto( _parse_response(response, ModelMetaParam()))
def register_model_version( self, model, model_path, project_snapshot_id=None, model_type=None, version_desc=None, current_stage=ModelVersionStage.GENERATED) -> ModelVersionMeta: """ register a model version in metadata store. :param model: Model id or model meta of registered model corresponded to model version :param model_path: Source path where the AIFlow model is stored. :param project_snapshot_id: Id of project snapshot corresponded to model version :param model_type: (Optional) Type of AIFlow registered model option. :param version_desc: (Optional) Description of registered model version. :param current_stage: (Optional) Stage of registered model version :return: A single :py:class:`ai_flow.meta.model_meta.ModelVersionMeta` object. """ if isinstance(model, int): model_id = model elif isinstance(model, ModelMeta): model_id = model.uuid else: raise Exception("can not recognize model {}".format(model)) model_version = ModelVersionProto( version=None, model_id=int64Value(model_id), project_snapshot_id=int64Value(project_snapshot_id), model_path=stringValue(model_path), model_type=stringValue(model_type), version_desc=stringValue(version_desc), current_stage=current_stage) request = metadata_service_pb2.RegisterModelVersionRequest( model_version=model_version) response = self.metadata_store_stub.registerModelVersion(request) return _unwrap_model_version_response(response)
def get_registered_model_detail( self, model_name) -> Optional[RegisteredModelDetail]: """ Get registered model detail filter by model name for Model Center. :param model_name: Name of registered model. This is expected to be unique in the backend store. :return: A single :py:class:`ai_flow.entities.model_registry.RegisteredModelDetail` object. """ request = GetRegisteredModelDetailRequest(model_meta=ModelMetaParam( model_name=stringValue(model_name))) response = self.model_center_stub.getRegisteredModelDetail(request) return RegisteredModelDetail.from_detail_proto( _parse_response(response, ProtoModelDetail()))
def update_registered_model( self, model_name, new_name=None, model_desc=None) -> Optional[RegisteredModelDetail]: """ Update metadata for RegisteredModel entity backend. Either ``model_name`` or ``model_desc`` should be non-None. Backend raises exception if a registered model with given name does not exist. :param model_name: Name of registered model. This is expected to be unique in the backend store. :param new_name: (Optional) New proposed name for the registered model. :param model_desc: (Optional) Description of registered model. :return: A single updated :py:class:`ai_flow.model_center.entity.RegisteredModelDetail` object. """ request = UpdateRegisteredModelRequest( model_meta=ModelMetaParam(model_name=stringValue(model_name)), registered_model=RegisteredModelParam( model_name=stringValue(new_name), model_desc=stringValue(model_desc))) response = self.model_center_stub.updateRegisteredModel(request) return RegisteredModelDetail.from_proto( _parse_response(response, RegisteredModelMeta()))
def update_project(self, project_name: Text, uri: Text = None, properties: Properties = None) -> Optional[ProjectMeta]: """ update project in metadata store. :param project_name: the name of the project :param uri: the uri of the project :param properties: the properties of the project :return: A single :py:class:`ai_flow.meta.project.ProjectMeta` object if update successfully. """ request = metadata_service_pb2.UpdateProjectRequest( name=project_name, uri=stringValue(uri), properties=properties) response = self.metadata_store_stub.updateProject(request) return _unwrap_project_response(response)
def update_dataset(self, dataset_name: Text, data_format: Text = None, description: Text = None, uri: Text = None, properties: Properties = None, name_list: List[Text] = None, type_list: List[DataType] = None, catalog_name: Text = None, catalog_type: Text = None, catalog_database: Text = None, catalog_connection_uri: Text = None, catalog_table: Text = None) -> Optional[DatasetMeta]: """ update dataset in metadata store. :param dataset_name: the name of the dataset :param data_format: the data format of the dataset :param description: the description of the dataset :param uri: the uri of the dataset :param properties: the properties of the dataset :param name_list: the name list of dataset's schema :param type_list: the type list corresponded to the name list of dataset's schema :param catalog_name: the name of the dataset catalog :param catalog_type: the type of the dataset catalog :param catalog_database: :param catalog_database: the database of the dataset catalog :param catalog_connection_uri: the connection uri of the dataset catalog :param catalog_table: the table of the dataset catalog :return: A single :py:class:`ai_flow.meta.dataset_meta.DatasetMeta` object if update successfully. """ request = metadata_service_pb2.UpdateDatasetRequest( name=dataset_name, data_format=stringValue(data_format), description=stringValue(description), uri=stringValue(uri), properties=properties, name_list=name_list, type_list=transform_dataset_type_list_to_proto(type_list), catalog_name=stringValue(catalog_name), catalog_type=stringValue(catalog_type), catalog_database=stringValue(catalog_database), catalog_connection_uri=stringValue(catalog_connection_uri), catalog_table=stringValue(catalog_table)) response = self.metadata_store_stub.updateDataset(request) return _unwrap_dataset_response(response)
def register_project(self, name, uri: Text = None, properties: Properties = None) -> ProjectMeta: """ register a project in metadata store. :param name: the name of the project :param uri: the uri of the project :param properties: the properties of the project :return: A single :py:class:`ai_flow.meta.project.ProjectMeta` object. """ project_request = ProjectProto(name=name, uri=stringValue(uri), properties=properties) request = metadata_service_pb2.RegisterProjectRequest( project=project_request) response = self.metadata_store_stub.registerProject(request) return _unwrap_project_response(response)
def register_model(self, model_name, project_id, model_desc=None) -> ModelMeta: """ register a model in metadata store :param model_name: Name of registered model :param project_id: Project id which registered model corresponded to. :param model_desc: Description of registered model :return: A single :py:class:`ai_flow.meta.model_meta.ModelMeta` object. """ model_request = ModelProto(name=model_name, model_desc=stringValue(model_desc), project_id=int64Value(project_id)) request = metadata_service_pb2.RegisterModelRequest( model=model_request) response = self.metadata_store_stub.registerModel(request) return _unwrap_model_response(response)
def register_dataset(self, name: Text, data_format: Text = None, description: Text = None, uri: Text = None, properties: Properties = None, name_list: List[Text] = None, type_list: List[DataType] = None) -> DatasetMeta: """ register an dataset in metadata store. :param name: the name of the dataset :param data_format: the data format of the dataset :param description: the description of the dataset :param uri: the uri of the dataset :param properties: the properties of the dataset :param name_list: the name list of dataset's schema :param type_list: the type list corresponded to the name list of dataset's schema :return: A single :py:class:`ai_flow.meta.dataset_meta.DatasetMeta` object. """ request = metadata_service_pb2.RegisterDatasetRequest( dataset=DatasetProto( name=name, data_format=stringValue(data_format), description=stringValue(description), uri=stringValue(uri), properties=properties, schema=SchemaProto( name_list=name_list, type_list=transform_dataset_type_list_to_proto(type_list)), catalog_name=stringValue(None), catalog_type=stringValue(None), catalog_database=stringValue(None), catalog_connection_uri=stringValue(None), catalog_table=stringValue(None))) response = self.metadata_store_stub.registerDataset(request) return _unwrap_dataset_response(response)
def register_model_version_relation( self, version, model_id, project_snapshot_id=None) -> ModelVersionRelationMeta: """ register a model version relation in metadata store. :param version: the specific model version :param model_id: the model id corresponded to the model version :param project_snapshot_id: the project snapshot id corresponded to the model version :return: A single :py:class:`ai_flow.meta.model_relation_meta.ModelVersionRelationMeta` object. """ model_version = ModelVersionRelationProto( version=stringValue(version), model_id=int64Value(model_id), project_snapshot_id=int64Value(project_snapshot_id)) request = metadata_service_pb2.RegisterModelVersionRelationRequest( model_version_relation=model_version) response = self.metadata_store_stub.registerModelVersionRelation( request) return _unwrap_model_version_relation_response(response)
def register_dataset_with_catalog( self, name: Text, catalog_name: Text, catalog_type: Text, catalog_connection_uri: Text, catalog_table: Text, catalog_database: Text = None) -> DatasetMeta: """ register dataset with catalog in metadata store. :param name: the name of the dataset :param catalog_name: the name of the dataset catalog :param catalog_type: the type of the dataset catalog :param catalog_connection_uri: the connection uri of the dataset catalog :param catalog_table: the table of the dataset catalog :param catalog_database: the database of the dataset catalog :return: A single :py:class:`ai_flow.meta.dataset_meta.DatasetMeta` object. """ request = metadata_service_pb2.RegisterDatasetRequest( dataset=DatasetProto( name=name, data_format=stringValue(None), description=stringValue(None), uri=stringValue(None), properties=None, schema=SchemaProto( name_list=None, type_list=transform_dataset_type_list_to_proto(None)), catalog_name=stringValue(catalog_name), catalog_type=stringValue(catalog_type), catalog_database=stringValue(catalog_database), catalog_connection_uri=stringValue(catalog_connection_uri), catalog_table=stringValue(catalog_table))) response = self.metadata_store_stub.registerDatasetWithCatalog(request) return _unwrap_dataset_response(response)