def run(
self,
dataset_name: str = None,
datastore: Datastore = None,
path: Union[str, List[str]] = None,
dataset_description: str = "",
dataset_tags: Optional[Dict[str, str]] = None,
include_path: bool = False,
set_column_types: Optional[Dict[str, DataType]] = None,
fine_grain_timestamp: str = None,
coarse_grain_timestamp: str = None,
partition_format: str = None,
create_new_version: bool = None,
) -> TabularDataset:
"""
Task run method.
Args:
- dataset_name (str, optional): The name of the Dataset in the Workspace
- datastore (azureml.core.datastore.Datastore, optional): The Datastore which holds
the files.
- path (Union[str, List[str]], optional): The path to the delimited files in the
Datastore.
- dataset_description (str, optional): Description of the Dataset.
- dataset_tags (str, optional): Tags to associate with the Dataset.
- include_path (bool, optional): Boolean to keep path information as column in the
dataset.
- set_column_types (Dict[str, azureml.data.DataType], optional): A dictionary to
set column data type, where key is column name and value is a
`azureml.data.DataType`.
- fine_grain_timestamp (str, optional): The name of column as fine grain timestamp.
- coarse_grain_timestamp (str, optional): The name of column coarse grain timestamp.
- partition_format (str, optional): Specify the partition format of path.
- create_new_version (bool, optional): Boolean to register the dataset as a new
version under the specified name.
Returns:
- azureml.data.TabularDataset: the created TabularDataset.
"""
if dataset_name is None:
raise ValueError("A dataset_name must be provided.")
if path is None:
raise ValueError("A path must be provided.")
if datastore is None:
raise ValueError("A path must be provided.")
if not isinstance(path, list):
path = [path]
dataset_tags = dataset_tags or dict()
dataset = azureml.core.dataset.Dataset.Tabular.from_parquet_files(
path=[(datastore, path_item) for path_item in path],
include_path=include_path,
set_column_types=set_column_types,
partition_format=partition_format,
)
dataset = dataset.with_timestamp_columns(
fine_grain_timestamp=fine_grain_timestamp,
coarse_grain_timestamp=coarse_grain_timestamp,
validate=True,
)
dataset = dataset.register(
workspace=datastore.workspace,
name=dataset_name,
description=dataset_description,
tags=dataset_tags,
create_new_version=create_new_version,
)
return dataset
def run(
self,
dataset_name: str = None,
datastore: Datastore = None,
path: Union[str, List[str]] = None,
dataset_description: str = "",
dataset_tags: Optional[Dict[str, str]] = None,
include_path: bool = False,
infer_column_types: bool = True,
set_column_types: Optional[Dict[str, DataType]] = None,
fine_grain_timestamp: str = None,
coarse_grain_timestamp: str = None,
separator: str = ",",
header: PromoteHeadersBehavior = PromoteHeadersBehavior.
ALL_FILES_HAVE_SAME_HEADERS,
partition_format: str = None,
create_new_version: bool = False,
) -> TabularDataset:
"""
Task run method.
Args:
- dataset_name (str, optional): The name of the Dataset in the Workspace
- datastore (azureml.core.datastore.Datastore, optional): The Datastore which holds
the files.
- path (Union[str, List[str]], optional): The path to the delimited files in the
Datastore.
- dataset_description (str, optional): Description of the Dataset.
- dataset_tags (str, optional): Tags to associate with the Dataset.
- include_path (bool, optional): Boolean to keep path information as column in the
dataset.
- infer_column_types (bool, optional): Boolean to infer column data types.
- set_column_types (Dict[str, azureml.data.DataType], optional): A dictionary to
set column data type, where key is column name and value is a
`azureml.data.DataType`.
- fine_grain_timestamp (str, optional): The name of column as fine grain timestamp.
- coarse_grain_timestamp (str, optional): The name of column coarse grain timestamp.
- separator (str, optional): The separator used to split columns.
- header (azureml.data.dataset_type_definitions.PromoteHeadersBehavior, optional):
Controls how column headers are promoted when reading from files. Defaults to
assume that all files have the same header.
- partition_format (str, optional): Specify the partition format of path.
- create_new_version (bool, optional): Boolean to register the dataset as a new
version under the specified name.
Returns:
- azureml.data.TabularDataset: the created TabularDataset
"""
if dataset_name is None:
raise ValueError("A dataset_name must be provided.")
if path is None:
raise ValueError("A path must be provided.")
if datastore is None:
raise ValueError("A path must be provided.")
if not isinstance(path, list):
path = [path]
dataset_tags = dataset_tags or dict()
dataset = azureml.core.dataset.Dataset.Tabular.from_delimited_files(
path=[(datastore, path_item) for path_item in path],
include_path=include_path,
infer_column_types=infer_column_types,
set_column_types=set_column_types,
separator=separator,
header=header,
partition_format=partition_format,
)
dataset = dataset.with_timestamp_columns(
fine_grain_timestamp=fine_grain_timestamp,
coarse_grain_timestamp=coarse_grain_timestamp,
validate=True,
)
dataset = dataset.register(
workspace=datastore.workspace,
name=dataset_name,
description=dataset_description,
tags=dataset_tags,
create_new_version=create_new_version,
)
return dataset