def dataset_meta_to_proto(dataset_mata) -> DatasetMeta:
if dataset_mata is None:
return None
else:
if dataset_mata.schema is not None:
name_list = dataset_mata.schema.name_list
type_list = dataset_mata.schema.type_list
data_type_list = []
if type_list is not None:
for data_type in type_list:
data_type_list.append(DataTypeProto.Value(data_type))
else:
data_type_list = None
else:
name_list = None
data_type_list = None
schema = SchemaProto(name_list=name_list,
type_list=data_type_list)
return DatasetProto(
uuid=dataset_mata.uuid,
name=dataset_mata.name,
properties=dataset_mata.properties,
data_format=stringValue(dataset_mata.data_format),
description=stringValue(dataset_mata.description),
uri=stringValue(dataset_mata.uri),
create_time=int64Value(dataset_mata.create_time),
update_time=int64Value(dataset_mata.update_time),
schema=schema,
catalog_name=stringValue(dataset_mata.catalog_name),
catalog_type=stringValue(dataset_mata.catalog_type),
catalog_database=stringValue(dataset_mata.catalog_database),
catalog_connection_uri=stringValue(dataset_mata.catalog_connection_uri),
catalog_table=stringValue(dataset_mata.catalog_table))
def workflow_execution_to_proto(workflow_execution: WorkflowExecutionInfo) -> WorkflowExecutionProto:
if workflow_execution is None:
return None
if workflow_execution.status is None:
state = Status.INIT
else:
state = workflow_execution.status
if workflow_execution.start_date is None:
start_date = '0'
else:
start_date = workflow_execution.start_date
if workflow_execution.end_date is None:
end_date = '0'
else:
end_date = workflow_execution.end_date
wp = WorkflowExecutionProto(execution_id=workflow_execution.workflow_execution_id,
execution_state=StateProto.Value(state),
start_time=int64Value(int(start_date)),
end_time=int64Value(int(end_date)),
workflow=workflow_to_proto(workflow_execution.workflow_info))
for k, v in workflow_execution.properties.items():
wp.properties[k] = v
return wp
def model_version_relation_meta_to_proto(
model_version_relation: ModelVersionRelationMeta) -> ModelVersionRelationProto:
if model_version_relation is None:
return None
else:
return ModelVersionRelationProto(
version=stringValue(model_version_relation.version),
model_id=int64Value(model_version_relation.model_id),
project_snapshot_id=int64Value(model_version_relation.project_snapshot_id))
def workflow_meta_to_proto(workflow_meta: WorkflowMeta) -> WorkflowMetaProto:
if workflow_meta is None:
return None
else:
return WorkflowMetaProto(
name=workflow_meta.name,
project_id=int64Value(workflow_meta.project_id),
properties=workflow_meta.properties,
create_time=int64Value(workflow_meta.create_time),
update_time=int64Value(workflow_meta.update_time),
uuid=workflow_meta.uuid)
def register_metric_meta(
self,
metric_name: Text,
metric_type: MetricType,
project_name: Text,
metric_desc: Optional[Text] = None,
dataset_name: Optional[Text] = None,
model_name: Optional[Text] = None,
job_name: Optional[Text] = None,
start_time: int = None,
end_time: int = None,
uri: Optional[Text] = None,
tags: Optional[Text] = None,
properties: Properties = None
) -> Tuple[int, Text, Optional[MetricMeta]]:
"""
Register metric metadata in Metric Center.
:param metric_name: Name of registered metric meta. This is expected to be unique in the backend store.
:param metric_type: Type of registered metric meta.
:param project_name: Name of the project associated with the registered metric meta.
:param metric_desc: (Optional) Description of registered metric meta.
:param dataset_name: (Optional) Name of the dataset associated with the registered metric meta.
:param model_name: (Optional) Name of the model associated with the registered metric meta.
:param job_name: (Optional) Name of the job associated with the registered metric meta.
:param start_time: (Optional) Start time of registered metric meta.
:param end_time: (Optional) End time of registered metric meta.
:param uri: (Optional) Uri of registered metric meta.
:param tags: (Optional) Tags of registered metric meta.
:param properties: (Optional) Properties of registered metric meta.
:return: A single :py:class:`ai_flow.meta.metric_meta.MetricMeta` object.
"""
request = MetricMetaRequest(metric_meta=MetricMetaProto(
metric_name=stringValue(metric_name),
metric_type=MetricTypeProto.Value(metric_type.value),
metric_desc=stringValue(metric_desc),
project_name=stringValue(project_name),
dataset_name=stringValue(dataset_name),
model_name=stringValue(model_name),
job_name=stringValue(job_name),
start_time=int64Value(start_time),
end_time=int64Value(end_time),
uri=stringValue(uri),
tags=stringValue(tags),
properties=properties))
response = self.metric_stub.registerMetricMeta(request)
if 0 == response.return_code:
metric_meta_proto = response.metric_meta
metric_meta = proto_to_metric_meta(metric_meta_proto)
return response.return_code, response.return_msg, metric_meta
else:
return response.return_code, response.return_msg, None
def model_version_store_to_proto(model_version_relation, model_center_detail) -> ModelVersionProto:
if model_version_relation is not None and model_center_detail is not None:
return ModelVersionProto(
version=stringValue(model_version_relation.version),
model_id=int64Value(model_version_relation.model_id),
project_snapshot_id=int64Value(model_version_relation.project_snapshot_id),
model_path=stringValue(model_center_detail.model_path),
model_type=stringValue(model_center_detail.model_type),
version_desc=stringValue(model_center_detail.version_desc),
current_stage=ModelVersionStage.Value(model_center_detail.current_stage.upper()))
else:
return None
def artifact_meta_to_proto(artifact_meta: ArtifactMeta) -> ArtifactProto:
if artifact_meta is None:
return None
else:
return ArtifactProto(
uuid=artifact_meta.uuid,
name=artifact_meta.name,
properties=artifact_meta.properties,
artifact_type=stringValue(artifact_meta.artifact_type),
description=stringValue(artifact_meta.description),
uri=stringValue(artifact_meta.uri),
create_time=int64Value(artifact_meta.create_time),
update_time=int64Value(artifact_meta.update_time))
def register_metric_summary(
self,
metric_name: Text,
metric_key: Text,
metric_value: Text,
metric_timestamp: int,
model_version: Optional[Text] = None,
job_execution_id: Optional[Text] = None
) -> Tuple[int, Text, Optional[MetricSummary]]:
"""
Register metric summary in Metric Center.
:param metric_name: Name of registered metric summary.
:param metric_key: Key of registered metric summary.
:param metric_value: Value of registered metric summary.
:param metric_timestamp: Timestamp of registered metric summary.
:param model_version: (Optional) Version of the model version associated with the registered metric summary.
:param job_execution_id: (Optional) ID of the job execution associated with the registered metric summary.
:return: A single :py:class:`ai_flow.meta.metric_meta.MetricSummary` object.
"""
request = MetricSummaryRequest(metric_summary=MetricSummaryProto(
metric_name=stringValue(metric_name),
metric_key=stringValue(metric_key),
metric_value=stringValue(metric_value),
metric_timestamp=int64Value(metric_timestamp),
model_version=stringValue(model_version),
job_execution_id=stringValue(job_execution_id)))
response = self.metric_stub.registerMetricSummary(request)
if 0 == response.return_code:
metric_summary_proto = response.metric_summary
metric_summary = proto_to_metric_summary(metric_summary_proto)
return response.return_code, response.return_msg, metric_summary
else:
return response.return_code, response.return_msg, None
def model_meta_to_proto(model_relation, model_center_detail) -> ModelProto:
if model_relation is not None and model_center_detail is not None:
return ModelProto(uuid=model_relation.uuid, name=model_relation.name,
project_id=int64Value(model_relation.project_id),
model_desc=stringValue(model_center_detail.model_desc))
else:
return None
def model_relation_meta_to_proto(model_relation_meta: ModelRelationMeta) -> ModelRelationProto:
if model_relation_meta is None:
return None
else:
return ModelRelationProto(
uuid=model_relation_meta.uuid,
name=model_relation_meta.name,
project_id=int64Value(model_relation_meta.project_id))
def list_metric_summaries(
self,
metric_name: Optional[Text] = None,
metric_key: Optional[Text] = None,
model_version: Optional[Text] = None,
start_time: int = None,
end_time=None
) -> Tuple[int, Text, Union[None, MetricSummary, List[MetricSummary]]]:
"""
List of metric summaries filter by metric summary fields for Metric Center.
:param metric_name: (Optional) Name of filtered metric summary.
:param metric_key: (Optional) Key of filtered metric summary.
:param model_version: (Optional) Version of the model version associated with the registered metric summary.
:param start_time: (Optional) Start time for timestamp filtered metric summary.
:param end_time: (Optional) End time for timestamp filtered metric summary.
:return: List of :py:class:`ai_flow.meta.metric_meta.MetricSummary` objects.
"""
request = ListMetricSummariesRequest(
metric_name=stringValue(metric_name),
metric_key=stringValue(metric_key),
model_version=stringValue(model_version),
start_time=int64Value(start_time),
end_time=int64Value(end_time))
response = self.metric_stub.listMetricSummaries(request)
if 0 == response.return_code:
repeated_metric_summary_proto = response.metric_summaries
if 1 == len(repeated_metric_summary_proto):
metric_summary = proto_to_metric_summary(
repeated_metric_summary_proto[0])
return response.return_code, response.return_msg, metric_summary
else:
res = []
for metric_summary_proto in repeated_metric_summary_proto:
res.append(proto_to_metric_summary(metric_summary_proto))
return response.return_code, response.return_msg, res
else:
return response.return_code, response.return_msg, None
def register_model_version_relation(
self,
version,
model_id,
project_snapshot_id=None) -> ModelVersionRelationMeta:
"""
register a model version relation in metadata store.
:param version: the specific model version
:param model_id: the model id corresponded to the model version
:param project_snapshot_id: the project snapshot id corresponded to the model version
:return: A single :py:class:`ai_flow.meta.model_relation_meta.ModelVersionRelationMeta` object.
"""
model_version = ModelVersionRelationProto(
version=stringValue(version),
model_id=int64Value(model_id),
project_snapshot_id=int64Value(project_snapshot_id))
request = metadata_service_pb2.RegisterModelVersionRelationRequest(
model_version_relation=model_version)
response = self.metadata_store_stub.registerModelVersionRelation(
request)
return _unwrap_model_version_relation_response(response)
def register_model_relation(self, name, project_id) -> ModelRelationMeta:
"""
register a model relation in metadata store
:param name: the name of the model
:param project_id: the project id which the model corresponded to.
:return: A single :py:class:`ai_flow.meta.model_relation_meta.ModelRelationMeta` object.
"""
request = metadata_service_pb2.RegisterModelRelationRequest(
model_relation=ModelRelationProto(
name=name, project_id=int64Value(project_id)))
response = self.metadata_store_stub.registerModelRelation(request)
return _unwrap_model_relation_response(response)
def register_model_version(
self,
model,
model_path,
project_snapshot_id=None,
model_type=None,
version_desc=None,
current_stage=ModelVersionStage.GENERATED) -> ModelVersionMeta:
"""
register a model version in metadata store.
:param model: Model id or model meta of registered model corresponded to model version
:param model_path: Source path where the AIFlow model is stored.
:param project_snapshot_id: Id of project snapshot corresponded to model version
:param model_type: (Optional) Type of AIFlow registered model option.
:param version_desc: (Optional) Description of registered model version.
:param current_stage: (Optional) Stage of registered model version
:return: A single :py:class:`ai_flow.meta.model_meta.ModelVersionMeta` object.
"""
if isinstance(model, int):
model_id = model
elif isinstance(model, ModelMeta):
model_id = model.uuid
else:
raise Exception("can not recognize model {}".format(model))
model_version = ModelVersionProto(
version=None,
model_id=int64Value(model_id),
project_snapshot_id=int64Value(project_snapshot_id),
model_path=stringValue(model_path),
model_type=stringValue(model_type),
version_desc=stringValue(version_desc),
current_stage=current_stage)
request = metadata_service_pb2.RegisterModelVersionRequest(
model_version=model_version)
response = self.metadata_store_stub.registerModelVersion(request)
return _unwrap_model_version_response(response)
def job_to_proto(job: JobExecutionInfo) -> JobProto:
if job.status is None:
state = Status.INIT
else:
state = job.status
if job.start_date is None:
start_date = '0'
else:
start_date = job.start_date
if job.end_date is None:
end_date = '0'
else:
end_date = job.end_date
return JobProto(name=job.job_name,
job_id=stringValue(job.job_execution_id),
job_state=StateProto.Value(state),
start_time=int64Value(int(start_date)),
end_time=int64Value(int(end_date)),
properties=dict(job.properties),
workflow_execution=workflow_execution_to_proto(job.workflow_execution))
def register_workflow(self,
name: Text,
project_id: int,
properties: Properties = None) -> WorkflowMeta:
"""
Register a workflow in metadata store.
:param name: the workflow name
:param project_id: the id of project which contains the workflow
:param properties: the workflow properties
"""
workflow_request = WorkflowMetaProto(name=name,
project_id=int64Value(project_id),
properties=properties)
request = metadata_service_pb2.RegisterWorkflowRequest(
workflow=workflow_request)
response = self.metadata_store_stub.registerWorkflow(request)
return _unwrap_workflow_response(response)
def register_model(self,
model_name,
project_id,
model_desc=None) -> ModelMeta:
"""
register a model in metadata store
:param model_name: Name of registered model
:param project_id: Project id which registered model corresponded to.
:param model_desc: Description of registered model
:return: A single :py:class:`ai_flow.meta.model_meta.ModelMeta` object.
"""
model_request = ModelProto(name=model_name,
model_desc=stringValue(model_desc),
project_id=int64Value(project_id))
request = metadata_service_pb2.RegisterModelRequest(
model=model_request)
response = self.metadata_store_stub.registerModel(request)
return _unwrap_model_response(response)
