def dataset_meta_to_proto(dataset_mata) -> DatasetMeta: if dataset_mata is None: return None else: if dataset_mata.schema is not None: name_list = dataset_mata.schema.name_list type_list = dataset_mata.schema.type_list data_type_list = [] if type_list is not None: for data_type in type_list: data_type_list.append(DataTypeProto.Value(data_type)) else: data_type_list = None else: name_list = None data_type_list = None schema = SchemaProto(name_list=name_list, type_list=data_type_list) return DatasetProto( uuid=dataset_mata.uuid, name=dataset_mata.name, properties=dataset_mata.properties, data_format=stringValue(dataset_mata.data_format), description=stringValue(dataset_mata.description), uri=stringValue(dataset_mata.uri), create_time=int64Value(dataset_mata.create_time), update_time=int64Value(dataset_mata.update_time), schema=schema, catalog_name=stringValue(dataset_mata.catalog_name), catalog_type=stringValue(dataset_mata.catalog_type), catalog_database=stringValue(dataset_mata.catalog_database), catalog_connection_uri=stringValue(dataset_mata.catalog_connection_uri), catalog_table=stringValue(dataset_mata.catalog_table))
def workflow_execution_to_proto(workflow_execution: WorkflowExecutionInfo) -> WorkflowExecutionProto: if workflow_execution is None: return None if workflow_execution.status is None: state = Status.INIT else: state = workflow_execution.status if workflow_execution.start_date is None: start_date = '0' else: start_date = workflow_execution.start_date if workflow_execution.end_date is None: end_date = '0' else: end_date = workflow_execution.end_date wp = WorkflowExecutionProto(execution_id=workflow_execution.workflow_execution_id, execution_state=StateProto.Value(state), start_time=int64Value(int(start_date)), end_time=int64Value(int(end_date)), workflow=workflow_to_proto(workflow_execution.workflow_info)) for k, v in workflow_execution.properties.items(): wp.properties[k] = v return wp
def model_version_relation_meta_to_proto( model_version_relation: ModelVersionRelationMeta) -> ModelVersionRelationProto: if model_version_relation is None: return None else: return ModelVersionRelationProto( version=stringValue(model_version_relation.version), model_id=int64Value(model_version_relation.model_id), project_snapshot_id=int64Value(model_version_relation.project_snapshot_id))
def workflow_meta_to_proto(workflow_meta: WorkflowMeta) -> WorkflowMetaProto: if workflow_meta is None: return None else: return WorkflowMetaProto( name=workflow_meta.name, project_id=int64Value(workflow_meta.project_id), properties=workflow_meta.properties, create_time=int64Value(workflow_meta.create_time), update_time=int64Value(workflow_meta.update_time), uuid=workflow_meta.uuid)
def register_metric_meta( self, metric_name: Text, metric_type: MetricType, project_name: Text, metric_desc: Optional[Text] = None, dataset_name: Optional[Text] = None, model_name: Optional[Text] = None, job_name: Optional[Text] = None, start_time: int = None, end_time: int = None, uri: Optional[Text] = None, tags: Optional[Text] = None, properties: Properties = None ) -> Tuple[int, Text, Optional[MetricMeta]]: """ Register metric metadata in Metric Center. :param metric_name: Name of registered metric meta. This is expected to be unique in the backend store. :param metric_type: Type of registered metric meta. :param project_name: Name of the project associated with the registered metric meta. :param metric_desc: (Optional) Description of registered metric meta. :param dataset_name: (Optional) Name of the dataset associated with the registered metric meta. :param model_name: (Optional) Name of the model associated with the registered metric meta. :param job_name: (Optional) Name of the job associated with the registered metric meta. :param start_time: (Optional) Start time of registered metric meta. :param end_time: (Optional) End time of registered metric meta. :param uri: (Optional) Uri of registered metric meta. :param tags: (Optional) Tags of registered metric meta. :param properties: (Optional) Properties of registered metric meta. :return: A single :py:class:`ai_flow.meta.metric_meta.MetricMeta` object. """ request = MetricMetaRequest(metric_meta=MetricMetaProto( metric_name=stringValue(metric_name), metric_type=MetricTypeProto.Value(metric_type.value), metric_desc=stringValue(metric_desc), project_name=stringValue(project_name), dataset_name=stringValue(dataset_name), model_name=stringValue(model_name), job_name=stringValue(job_name), start_time=int64Value(start_time), end_time=int64Value(end_time), uri=stringValue(uri), tags=stringValue(tags), properties=properties)) response = self.metric_stub.registerMetricMeta(request) if 0 == response.return_code: metric_meta_proto = response.metric_meta metric_meta = proto_to_metric_meta(metric_meta_proto) return response.return_code, response.return_msg, metric_meta else: return response.return_code, response.return_msg, None
def model_version_store_to_proto(model_version_relation, model_center_detail) -> ModelVersionProto: if model_version_relation is not None and model_center_detail is not None: return ModelVersionProto( version=stringValue(model_version_relation.version), model_id=int64Value(model_version_relation.model_id), project_snapshot_id=int64Value(model_version_relation.project_snapshot_id), model_path=stringValue(model_center_detail.model_path), model_type=stringValue(model_center_detail.model_type), version_desc=stringValue(model_center_detail.version_desc), current_stage=ModelVersionStage.Value(model_center_detail.current_stage.upper())) else: return None
def artifact_meta_to_proto(artifact_meta: ArtifactMeta) -> ArtifactProto: if artifact_meta is None: return None else: return ArtifactProto( uuid=artifact_meta.uuid, name=artifact_meta.name, properties=artifact_meta.properties, artifact_type=stringValue(artifact_meta.artifact_type), description=stringValue(artifact_meta.description), uri=stringValue(artifact_meta.uri), create_time=int64Value(artifact_meta.create_time), update_time=int64Value(artifact_meta.update_time))
def register_metric_summary( self, metric_name: Text, metric_key: Text, metric_value: Text, metric_timestamp: int, model_version: Optional[Text] = None, job_execution_id: Optional[Text] = None ) -> Tuple[int, Text, Optional[MetricSummary]]: """ Register metric summary in Metric Center. :param metric_name: Name of registered metric summary. :param metric_key: Key of registered metric summary. :param metric_value: Value of registered metric summary. :param metric_timestamp: Timestamp of registered metric summary. :param model_version: (Optional) Version of the model version associated with the registered metric summary. :param job_execution_id: (Optional) ID of the job execution associated with the registered metric summary. :return: A single :py:class:`ai_flow.meta.metric_meta.MetricSummary` object. """ request = MetricSummaryRequest(metric_summary=MetricSummaryProto( metric_name=stringValue(metric_name), metric_key=stringValue(metric_key), metric_value=stringValue(metric_value), metric_timestamp=int64Value(metric_timestamp), model_version=stringValue(model_version), job_execution_id=stringValue(job_execution_id))) response = self.metric_stub.registerMetricSummary(request) if 0 == response.return_code: metric_summary_proto = response.metric_summary metric_summary = proto_to_metric_summary(metric_summary_proto) return response.return_code, response.return_msg, metric_summary else: return response.return_code, response.return_msg, None
def model_meta_to_proto(model_relation, model_center_detail) -> ModelProto: if model_relation is not None and model_center_detail is not None: return ModelProto(uuid=model_relation.uuid, name=model_relation.name, project_id=int64Value(model_relation.project_id), model_desc=stringValue(model_center_detail.model_desc)) else: return None
def model_relation_meta_to_proto(model_relation_meta: ModelRelationMeta) -> ModelRelationProto: if model_relation_meta is None: return None else: return ModelRelationProto( uuid=model_relation_meta.uuid, name=model_relation_meta.name, project_id=int64Value(model_relation_meta.project_id))
def list_metric_summaries( self, metric_name: Optional[Text] = None, metric_key: Optional[Text] = None, model_version: Optional[Text] = None, start_time: int = None, end_time=None ) -> Tuple[int, Text, Union[None, MetricSummary, List[MetricSummary]]]: """ List of metric summaries filter by metric summary fields for Metric Center. :param metric_name: (Optional) Name of filtered metric summary. :param metric_key: (Optional) Key of filtered metric summary. :param model_version: (Optional) Version of the model version associated with the registered metric summary. :param start_time: (Optional) Start time for timestamp filtered metric summary. :param end_time: (Optional) End time for timestamp filtered metric summary. :return: List of :py:class:`ai_flow.meta.metric_meta.MetricSummary` objects. """ request = ListMetricSummariesRequest( metric_name=stringValue(metric_name), metric_key=stringValue(metric_key), model_version=stringValue(model_version), start_time=int64Value(start_time), end_time=int64Value(end_time)) response = self.metric_stub.listMetricSummaries(request) if 0 == response.return_code: repeated_metric_summary_proto = response.metric_summaries if 1 == len(repeated_metric_summary_proto): metric_summary = proto_to_metric_summary( repeated_metric_summary_proto[0]) return response.return_code, response.return_msg, metric_summary else: res = [] for metric_summary_proto in repeated_metric_summary_proto: res.append(proto_to_metric_summary(metric_summary_proto)) return response.return_code, response.return_msg, res else: return response.return_code, response.return_msg, None
def register_model_version_relation( self, version, model_id, project_snapshot_id=None) -> ModelVersionRelationMeta: """ register a model version relation in metadata store. :param version: the specific model version :param model_id: the model id corresponded to the model version :param project_snapshot_id: the project snapshot id corresponded to the model version :return: A single :py:class:`ai_flow.meta.model_relation_meta.ModelVersionRelationMeta` object. """ model_version = ModelVersionRelationProto( version=stringValue(version), model_id=int64Value(model_id), project_snapshot_id=int64Value(project_snapshot_id)) request = metadata_service_pb2.RegisterModelVersionRelationRequest( model_version_relation=model_version) response = self.metadata_store_stub.registerModelVersionRelation( request) return _unwrap_model_version_relation_response(response)
def register_model_relation(self, name, project_id) -> ModelRelationMeta: """ register a model relation in metadata store :param name: the name of the model :param project_id: the project id which the model corresponded to. :return: A single :py:class:`ai_flow.meta.model_relation_meta.ModelRelationMeta` object. """ request = metadata_service_pb2.RegisterModelRelationRequest( model_relation=ModelRelationProto( name=name, project_id=int64Value(project_id))) response = self.metadata_store_stub.registerModelRelation(request) return _unwrap_model_relation_response(response)
def register_model_version( self, model, model_path, project_snapshot_id=None, model_type=None, version_desc=None, current_stage=ModelVersionStage.GENERATED) -> ModelVersionMeta: """ register a model version in metadata store. :param model: Model id or model meta of registered model corresponded to model version :param model_path: Source path where the AIFlow model is stored. :param project_snapshot_id: Id of project snapshot corresponded to model version :param model_type: (Optional) Type of AIFlow registered model option. :param version_desc: (Optional) Description of registered model version. :param current_stage: (Optional) Stage of registered model version :return: A single :py:class:`ai_flow.meta.model_meta.ModelVersionMeta` object. """ if isinstance(model, int): model_id = model elif isinstance(model, ModelMeta): model_id = model.uuid else: raise Exception("can not recognize model {}".format(model)) model_version = ModelVersionProto( version=None, model_id=int64Value(model_id), project_snapshot_id=int64Value(project_snapshot_id), model_path=stringValue(model_path), model_type=stringValue(model_type), version_desc=stringValue(version_desc), current_stage=current_stage) request = metadata_service_pb2.RegisterModelVersionRequest( model_version=model_version) response = self.metadata_store_stub.registerModelVersion(request) return _unwrap_model_version_response(response)
def job_to_proto(job: JobExecutionInfo) -> JobProto: if job.status is None: state = Status.INIT else: state = job.status if job.start_date is None: start_date = '0' else: start_date = job.start_date if job.end_date is None: end_date = '0' else: end_date = job.end_date return JobProto(name=job.job_name, job_id=stringValue(job.job_execution_id), job_state=StateProto.Value(state), start_time=int64Value(int(start_date)), end_time=int64Value(int(end_date)), properties=dict(job.properties), workflow_execution=workflow_execution_to_proto(job.workflow_execution))
def register_workflow(self, name: Text, project_id: int, properties: Properties = None) -> WorkflowMeta: """ Register a workflow in metadata store. :param name: the workflow name :param project_id: the id of project which contains the workflow :param properties: the workflow properties """ workflow_request = WorkflowMetaProto(name=name, project_id=int64Value(project_id), properties=properties) request = metadata_service_pb2.RegisterWorkflowRequest( workflow=workflow_request) response = self.metadata_store_stub.registerWorkflow(request) return _unwrap_workflow_response(response)
def register_model(self, model_name, project_id, model_desc=None) -> ModelMeta: """ register a model in metadata store :param model_name: Name of registered model :param project_id: Project id which registered model corresponded to. :param model_desc: Description of registered model :return: A single :py:class:`ai_flow.meta.model_meta.ModelMeta` object. """ model_request = ModelProto(name=model_name, model_desc=stringValue(model_desc), project_id=int64Value(project_id)) request = metadata_service_pb2.RegisterModelRequest( model=model_request) response = self.metadata_store_stub.registerModel(request) return _unwrap_model_response(response)