def get_dataset_by_name(self, dataset_name) -> Optional[DatasetMeta]:
"""
get a specific dataset in metadata store by dataset name.
:param dataset_name: the dataset name
:return: A single :py:class:`ai_flow.meta.dataset_meta.DatasetMeta` object if the dataset exists,,
Otherwise, returns None if the dataset does not exist.
"""
request = metadata_service_pb2.NameRequest(name=dataset_name)
response = self.metadata_store_stub.getDatasetByName(request)
return _unwrap_dataset_response(response)
def update_dataset(self,
dataset_name: Text,
data_format: Text = None,
description: Text = None,
uri: Text = None,
properties: Properties = None,
name_list: List[Text] = None,
type_list: List[DataType] = None,
catalog_name: Text = None,
catalog_type: Text = None,
catalog_database: Text = None,
catalog_connection_uri: Text = None,
catalog_table: Text = None) -> Optional[DatasetMeta]:
"""
update dataset in metadata store.
:param dataset_name: the name of the dataset
:param data_format: the data format of the dataset
:param description: the description of the dataset
:param uri: the uri of the dataset
:param properties: the properties of the dataset
:param name_list: the name list of dataset's schema
:param type_list: the type list corresponded to the name list of dataset's schema
:param catalog_name: the name of the dataset catalog
:param catalog_type: the type of the dataset catalog
:param catalog_database: :param catalog_database: the database of the dataset catalog
:param catalog_connection_uri: the connection uri of the dataset catalog
:param catalog_table: the table of the dataset catalog
:return: A single :py:class:`ai_flow.meta.dataset_meta.DatasetMeta` object if update successfully.
"""
request = metadata_service_pb2.UpdateDatasetRequest(
name=dataset_name,
data_format=stringValue(data_format),
description=stringValue(description),
uri=stringValue(uri),
properties=properties,
name_list=name_list,
type_list=transform_dataset_type_list_to_proto(type_list),
catalog_name=stringValue(catalog_name),
catalog_type=stringValue(catalog_type),
catalog_database=stringValue(catalog_database),
catalog_connection_uri=stringValue(catalog_connection_uri),
catalog_table=stringValue(catalog_table))
response = self.metadata_store_stub.updateDataset(request)
return _unwrap_dataset_response(response)
def register_dataset(self,
name: Text,
data_format: Text = None,
description: Text = None,
uri: Text = None,
properties: Properties = None,
name_list: List[Text] = None,
type_list: List[DataType] = None) -> DatasetMeta:
"""
register an dataset in metadata store.
:param name: the name of the dataset
:param data_format: the data format of the dataset
:param description: the description of the dataset
:param uri: the uri of the dataset
:param properties: the properties of the dataset
:param name_list: the name list of dataset's schema
:param type_list: the type list corresponded to the name list of dataset's schema
:return: A single :py:class:`ai_flow.meta.dataset_meta.DatasetMeta` object.
"""
request = metadata_service_pb2.RegisterDatasetRequest(
dataset=DatasetProto(
name=name,
data_format=stringValue(data_format),
description=stringValue(description),
uri=stringValue(uri),
properties=properties,
schema=SchemaProto(
name_list=name_list,
type_list=transform_dataset_type_list_to_proto(type_list)),
catalog_name=stringValue(None),
catalog_type=stringValue(None),
catalog_database=stringValue(None),
catalog_connection_uri=stringValue(None),
catalog_table=stringValue(None)))
response = self.metadata_store_stub.registerDataset(request)
return _unwrap_dataset_response(response)
def register_dataset_with_catalog(
self,
name: Text,
catalog_name: Text,
catalog_type: Text,
catalog_connection_uri: Text,
catalog_table: Text,
catalog_database: Text = None) -> DatasetMeta:
"""
register dataset with catalog in metadata store.
:param name: the name of the dataset
:param catalog_name: the name of the dataset catalog
:param catalog_type: the type of the dataset catalog
:param catalog_connection_uri: the connection uri of the dataset catalog
:param catalog_table: the table of the dataset catalog
:param catalog_database: the database of the dataset catalog
:return: A single :py:class:`ai_flow.meta.dataset_meta.DatasetMeta` object.
"""
request = metadata_service_pb2.RegisterDatasetRequest(
dataset=DatasetProto(
name=name,
data_format=stringValue(None),
description=stringValue(None),
uri=stringValue(None),
properties=None,
schema=SchemaProto(
name_list=None,
type_list=transform_dataset_type_list_to_proto(None)),
catalog_name=stringValue(catalog_name),
catalog_type=stringValue(catalog_type),
catalog_database=stringValue(catalog_database),
catalog_connection_uri=stringValue(catalog_connection_uri),
catalog_table=stringValue(catalog_table)))
response = self.metadata_store_stub.registerDatasetWithCatalog(request)
return _unwrap_dataset_response(response)