def _read_text(
parser_func: Callable[..., pd.DataFrame],
path: Union[str, List[str]],
path_suffix: Union[str, List[str], None],
path_ignore_suffix: Union[str, List[str], None],
ignore_empty: bool,
use_threads: bool,
last_modified_begin: Optional[datetime.datetime],
last_modified_end: Optional[datetime.datetime],
boto3_session: Optional[boto3.Session],
s3_additional_kwargs: Optional[Dict[str, str]],
chunksize: Optional[int],
dataset: bool,
partition_filter: Optional[Callable[[Dict[str, str]], bool]],
ignore_index: bool,
**pandas_kwargs: Any,
) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
if "iterator" in pandas_kwargs:
raise exceptions.InvalidArgument(
"Please, use the chunksize argument instead of iterator.")
session: boto3.Session = _utils.ensure_session(session=boto3_session)
paths: List[str] = _path2list(
path=path,
boto3_session=session,
suffix=path_suffix,
ignore_suffix=_get_path_ignore_suffix(
path_ignore_suffix=path_ignore_suffix),
ignore_empty=ignore_empty,
last_modified_begin=last_modified_begin,
last_modified_end=last_modified_end,
s3_additional_kwargs=s3_additional_kwargs,
)
path_root: Optional[str] = _get_path_root(path=path, dataset=dataset)
if path_root is not None:
paths = _apply_partition_filter(path_root=path_root,
paths=paths,
filter_func=partition_filter)
if len(paths) < 1:
raise exceptions.NoFilesFound(f"No files Found on: {path}.")
_logger.debug("paths:\n%s", paths)
args: Dict[str, Any] = {
"parser_func": parser_func,
"boto3_session": session,
"dataset": dataset,
"path_root": path_root,
"pandas_kwargs": pandas_kwargs,
"s3_additional_kwargs": s3_additional_kwargs,
"use_threads": use_threads,
}
_logger.debug("args:\n%s", pprint.pformat(args))
ret: Union[pd.DataFrame, Iterator[pd.DataFrame]]
if chunksize is not None:
ret = _read_text_chunked(paths=paths, chunksize=chunksize, **args)
elif len(paths) == 1:
ret = _read_text_file(path=paths[0], **args)
else:
ret = _union(dfs=[_read_text_file(path=p, **args) for p in paths],
ignore_index=ignore_index)
return ret
def _read_parquet_metadata(
path: Union[str, List[str]],
path_suffix: Optional[str],
path_ignore_suffix: Optional[str],
ignore_empty: bool,
ignore_null: bool,
dtype: Optional[Dict[str, str]],
sampling: float,
dataset: bool,
use_threads: Union[bool, int],
boto3_session: boto3.Session,
s3_additional_kwargs: Optional[Dict[str, str]],
version_id: Optional[Union[str, Dict[str, str]]] = None,
pyarrow_additional_kwargs: Optional[Dict[str, Any]] = None,
) -> Tuple[Dict[str, str], Optional[Dict[str, str]], Optional[Dict[str, List[str]]]]:
"""Handle wr.s3.read_parquet_metadata internally."""
path_root: Optional[str] = _get_path_root(path=path, dataset=dataset)
paths: List[str] = _path2list(
path=path,
boto3_session=boto3_session,
suffix=path_suffix,
ignore_suffix=_get_path_ignore_suffix(path_ignore_suffix=path_ignore_suffix),
ignore_empty=ignore_empty,
s3_additional_kwargs=s3_additional_kwargs,
)
# Files
schemas: Tuple[Dict[str, str], ...] = _read_schemas_from_files(
paths=paths,
sampling=sampling,
use_threads=use_threads,
boto3_session=boto3_session,
s3_additional_kwargs=s3_additional_kwargs,
version_ids=version_id
if isinstance(version_id, dict)
else {paths[0]: version_id}
if isinstance(version_id, str)
else None,
ignore_null=ignore_null,
pyarrow_additional_kwargs=pyarrow_additional_kwargs,
)
columns_types: Dict[str, str] = _merge_schemas(schemas=schemas)
# Partitions
partitions_types: Optional[Dict[str, str]] = None
partitions_values: Optional[Dict[str, List[str]]] = None
if (dataset is True) and (path_root is not None):
partitions_types, partitions_values = _extract_partitions_metadata_from_paths(path=path_root, paths=paths)
# Casting
if dtype:
for k, v in dtype.items():
if columns_types and k in columns_types:
columns_types[k] = v
if partitions_types and k in partitions_types:
partitions_types[k] = v
return columns_types, partitions_types, partitions_values
def read_parquet(
path: Union[str, List[str]],
path_suffix: Union[str, List[str], None] = None,
path_ignore_suffix: Union[str, List[str], None] = None,
partition_filter: Optional[Callable[[Dict[str, str]], bool]] = None,
columns: Optional[List[str]] = None,
validate_schema: bool = False,
chunked: Union[bool, int] = False,
dataset: bool = False,
categories: Optional[List[str]] = None,
safe: bool = True,
use_threads: bool = True,
last_modified_begin: Optional[datetime.datetime] = None,
last_modified_end: Optional[datetime.datetime] = None,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, Any]] = None,
) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
"""Read Apache Parquet file(s) from from a received S3 prefix or list of S3 objects paths.
The concept of Dataset goes beyond the simple idea of files and enable more
complex features like partitioning and catalog integration (AWS Glue Catalog).
This function accepts Unix shell-style wildcards in the path argument.
* (matches everything), ? (matches any single character),
[seq] (matches any character in seq), [!seq] (matches any character not in seq).
Note
----
``Batching`` (`chunked` argument) (Memory Friendly):
Will anable the function to return a Iterable of DataFrames instead of a regular DataFrame.
There are two batching strategies on Wrangler:
- If **chunked=True**, a new DataFrame will be returned for each file in your path/dataset.
- If **chunked=INTEGER**, Wrangler will iterate on the data by number of rows igual the received INTEGER.
`P.S.` `chunked=True` if faster and uses less memory while `chunked=INTEGER` is more precise
in number of rows for each Dataframe.
Note
----
In case of `use_threads=True` the number of threads
that will be spawned will be gotten from os.cpu_count().
Note
----
The filter by last_modified begin last_modified end is applied after list all S3 files
Parameters
----------
path : Union[str, List[str]]
S3 prefix (accepts Unix shell-style wildcards)
(e.g. s3://bucket/prefix) or list of S3 objects paths (e.g. [s3://bucket/key0, s3://bucket/key1]).
path_suffix: Union[str, List[str], None]
Suffix or List of suffixes for filtering S3 keys.
path_ignore_suffix: Union[str, List[str], None]
Suffix or List of suffixes for S3 keys to be ignored.
partition_filter: Optional[Callable[[Dict[str, str]], bool]]
Callback Function filters to apply on PARTITION columns (PUSH-DOWN filter).
This function MUST receive a single argument (Dict[str, str]) where keys are partitions
names and values are partitions values. Partitions values will be always strings extracted from S3.
This function MUST return a bool, True to read the partition or False to ignore it.
Ignored if `dataset=False`.
E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False``
columns : List[str], optional
Names of columns to read from the file(s).
validate_schema:
Check that individual file schemas are all the same / compatible. Schemas within a
folder prefix should all be the same. Disable if you have schemas that are different
and want to disable this check.
chunked : Union[int, bool]
If passed will split the data in a Iterable of DataFrames (Memory friendly).
If `True` wrangler will iterate on the data by files in the most efficient way without guarantee of chunksize.
If an `INTEGER` is passed Wrangler will iterate on the data by number of rows igual the received INTEGER.
dataset: bool
If `True` read a parquet dataset instead of simple file(s) loading all the related partitions as columns.
categories: Optional[List[str]], optional
List of columns names that should be returned as pandas.Categorical.
Recommended for memory restricted environments.
safe : bool, default True
For certain data types, a cast is needed in order to store the
data in a pandas DataFrame or Series (e.g. timestamps are always
stored as nanoseconds in pandas). This option controls whether it
is a safe cast or not.
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
last_modified_begin
Filter the s3 files by the Last modified date of the object.
The filter is applied only after list all s3 files.
last_modified_end: datetime, optional
Filter the s3 files by the Last modified date of the object.
The filter is applied only after list all s3 files.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
s3_additional_kwargs : Optional[Dict[str, Any]]
Forward to botocore requests, only "SSECustomerAlgorithm" and "SSECustomerKey" arguments will be considered.
Returns
-------
Union[pandas.DataFrame, Generator[pandas.DataFrame, None, None]]
Pandas DataFrame or a Generator in case of `chunked=True`.
Examples
--------
Reading all Parquet files under a prefix
>>> import awswrangler as wr
>>> df = wr.s3.read_parquet(path='s3://bucket/prefix/')
Reading all Parquet files from a list
>>> import awswrangler as wr
>>> df = wr.s3.read_parquet(path=['s3://bucket/filename0.parquet', 's3://bucket/filename1.parquet'])
Reading in chunks (Chunk by file)
>>> import awswrangler as wr
>>> dfs = wr.s3.read_parquet(path=['s3://bucket/filename0.csv', 's3://bucket/filename1.csv'], chunked=True)
>>> for df in dfs:
>>> print(df) # Smaller Pandas DataFrame
Reading in chunks (Chunk by 1MM rows)
>>> import awswrangler as wr
>>> dfs = wr.s3.read_parquet(path=['s3://bucket/filename0.csv', 's3://bucket/filename1.csv'], chunked=1_000_000)
>>> for df in dfs:
>>> print(df) # 1MM Pandas DataFrame
Reading Parquet Dataset with PUSH-DOWN filter over partitions
>>> import awswrangler as wr
>>> my_filter = lambda x: True if x["city"].startswith("new") else False
>>> df = wr.s3.read_parquet(path, dataset=True, partition_filter=my_filter)
"""
session: boto3.Session = _utils.ensure_session(session=boto3_session)
paths: List[str] = _path2list(
path=path,
boto3_session=session,
suffix=path_suffix,
ignore_suffix=_get_path_ignore_suffix(
path_ignore_suffix=path_ignore_suffix),
last_modified_begin=last_modified_begin,
last_modified_end=last_modified_end,
)
path_root: Optional[str] = _get_path_root(path=path, dataset=dataset)
if path_root is not None:
paths = _apply_partition_filter(path_root=path_root,
paths=paths,
filter_func=partition_filter)
if len(paths) < 1:
raise exceptions.NoFilesFound(f"No files Found on: {path}.")
_logger.debug("paths:\n%s", paths)
args: Dict[str, Any] = {
"columns": columns,
"categories": categories,
"safe": safe,
"boto3_session": session,
"dataset": dataset,
"path_root": path_root,
"s3_additional_kwargs": s3_additional_kwargs,
"use_threads": use_threads,
}
_logger.debug("args:\n%s", pprint.pformat(args))
if chunked is not False:
return _read_parquet_chunked(paths=paths,
chunked=chunked,
validate_schema=validate_schema,
**args)
if len(paths) == 1:
return _read_parquet(path=paths[0], **args)
if validate_schema is True:
_validate_schemas_from_files(
paths=paths,
sampling=1.0,
use_threads=True,
boto3_session=boto3_session,
s3_additional_kwargs=s3_additional_kwargs,
)
return _union(dfs=[_read_parquet(path=p, **args) for p in paths],
ignore_index=None)
def read_parquet(
path: Union[str, List[str]],
path_root: Optional[str] = None,
path_suffix: Union[str, List[str], None] = None,
path_ignore_suffix: Union[str, List[str], None] = None,
version_id: Optional[Union[str, Dict[str, str]]] = None,
ignore_empty: bool = True,
ignore_index: Optional[bool] = None,
partition_filter: Optional[Callable[[Dict[str, str]], bool]] = None,
columns: Optional[List[str]] = None,
validate_schema: bool = False,
chunked: Union[bool, int] = False,
dataset: bool = False,
categories: Optional[List[str]] = None,
safe: bool = True,
map_types: bool = True,
use_threads: Union[bool, int] = True,
last_modified_begin: Optional[datetime.datetime] = None,
last_modified_end: Optional[datetime.datetime] = None,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, Any]] = None,
pyarrow_additional_kwargs: Optional[Dict[str, Any]] = None,
) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
"""Read Apache Parquet file(s) from from a received S3 prefix or list of S3 objects paths.
The concept of Dataset goes beyond the simple idea of files and enable more
complex features like partitioning and catalog integration (AWS Glue Catalog).
This function accepts Unix shell-style wildcards in the path argument.
* (matches everything), ? (matches any single character),
[seq] (matches any character in seq), [!seq] (matches any character not in seq).
If you want to use a path which includes Unix shell-style wildcard characters (`*, ?, []`),
you can use `glob.escape(path)` before passing the path to this function.
Note
----
``Batching`` (`chunked` argument) (Memory Friendly):
Will anable the function to return a Iterable of DataFrames instead of a regular DataFrame.
There are two batching strategies on Wrangler:
- If **chunked=True**, a new DataFrame will be returned for each file in your path/dataset.
- If **chunked=INTEGER**, Wrangler will iterate on the data by number of rows igual the received INTEGER.
`P.S.` `chunked=True` if faster and uses less memory while `chunked=INTEGER` is more precise
in number of rows for each Dataframe.
Note
----
In case of `use_threads=True` the number of threads
that will be spawned will be gotten from os.cpu_count().
Note
----
The filter by last_modified begin last_modified end is applied after list all S3 files
Parameters
----------
path : Union[str, List[str]]
S3 prefix (accepts Unix shell-style wildcards)
(e.g. s3://bucket/prefix) or list of S3 objects paths (e.g. [s3://bucket/key0, s3://bucket/key1]).
path_root : Optional[str]
Root path of the table. If dataset=`True`, will be used as a starting point to load partition columns.
path_suffix: Union[str, List[str], None]
Suffix or List of suffixes to be read (e.g. [".gz.parquet", ".snappy.parquet"]).
If None, will try to read all files. (default)
path_ignore_suffix: Union[str, List[str], None]
Suffix or List of suffixes for S3 keys to be ignored.(e.g. [".csv", "_SUCCESS"]).
If None, will try to read all files. (default)
version_id: Optional[Union[str, Dict[str, str]]]
Version id of the object or mapping of object path to version id.
(e.g. {'s3://bucket/key0': '121212', 's3://bucket/key1': '343434'})
ignore_empty: bool
Ignore files with 0 bytes.
ignore_index: Optional[bool]
Ignore index when combining multiple parquet files to one DataFrame.
partition_filter: Optional[Callable[[Dict[str, str]], bool]]
Callback Function filters to apply on PARTITION columns (PUSH-DOWN filter).
This function MUST receive a single argument (Dict[str, str]) where keys are partitions
names and values are partitions values. Partitions values will be always strings extracted from S3.
This function MUST return a bool, True to read the partition or False to ignore it.
Ignored if `dataset=False`.
E.g ``lambda x: True if x["year"] == "2020" and x["month"] == "1" else False``
columns : List[str], optional
Names of columns to read from the file(s).
validate_schema:
Check that individual file schemas are all the same / compatible. Schemas within a
folder prefix should all be the same. Disable if you have schemas that are different
and want to disable this check.
chunked : Union[int, bool]
If passed will split the data in a Iterable of DataFrames (Memory friendly).
If `True` wrangler will iterate on the data by files in the most efficient way without guarantee of chunksize.
If an `INTEGER` is passed Wrangler will iterate on the data by number of rows igual the received INTEGER.
dataset: bool
If `True` read a parquet dataset instead of simple file(s) loading all the related partitions as columns.
categories: Optional[List[str]], optional
List of columns names that should be returned as pandas.Categorical.
Recommended for memory restricted environments.
safe : bool, default True
For certain data types, a cast is needed in order to store the
data in a pandas DataFrame or Series (e.g. timestamps are always
stored as nanoseconds in pandas). This option controls whether it
is a safe cast or not.
map_types : bool, default True
True to convert pyarrow DataTypes to pandas ExtensionDtypes. It is
used to override the default pandas type for conversion of built-in
pyarrow types or in absence of pandas_metadata in the Table schema.
use_threads : Union[bool, int]
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
If integer is provided, specified number is used.
last_modified_begin
Filter the s3 files by the Last modified date of the object.
The filter is applied only after list all s3 files.
last_modified_end: datetime, optional
Filter the s3 files by the Last modified date of the object.
The filter is applied only after list all s3 files.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
s3_additional_kwargs : Optional[Dict[str, Any]]
Forward to botocore requests, only "SSECustomerAlgorithm" and "SSECustomerKey" arguments will be considered.
pyarrow_additional_kwargs : Optional[Dict[str, Any]]
Forward to the ParquetFile class or converting an Arrow table to Pandas, currently only an
"coerce_int96_timestamp_unit" or "timestamp_as_object" argument will be considered. If reading parquet
files where you cannot convert a timestamp to pandas Timestamp[ns] consider setting timestamp_as_object=True,
to allow for timestamp units larger than "ns". If reading parquet data that still uses INT96 (like Athena
outputs) you can use coerce_int96_timestamp_unit to specify what timestamp unit to encode INT96 to (by default
this is "ns", if you know the output parquet came from a system that encodes timestamp to a particular unit
then set this to that same unit e.g. coerce_int96_timestamp_unit="ms").
Returns
-------
Union[pandas.DataFrame, Generator[pandas.DataFrame, None, None]]
Pandas DataFrame or a Generator in case of `chunked=True`.
Examples
--------
Reading all Parquet files under a prefix
>>> import awswrangler as wr
>>> df = wr.s3.read_parquet(path='s3://bucket/prefix/')
Reading all Parquet files from a list
>>> import awswrangler as wr
>>> df = wr.s3.read_parquet(path=['s3://bucket/filename0.parquet', 's3://bucket/filename1.parquet'])
Reading in chunks (Chunk by file)
>>> import awswrangler as wr
>>> dfs = wr.s3.read_parquet(path=['s3://bucket/filename0.csv', 's3://bucket/filename1.csv'], chunked=True)
>>> for df in dfs:
>>> print(df) # Smaller Pandas DataFrame
Reading in chunks (Chunk by 1MM rows)
>>> import awswrangler as wr
>>> dfs = wr.s3.read_parquet(path=['s3://bucket/filename0.csv', 's3://bucket/filename1.csv'], chunked=1_000_000)
>>> for df in dfs:
>>> print(df) # 1MM Pandas DataFrame
Reading Parquet Dataset with PUSH-DOWN filter over partitions
>>> import awswrangler as wr
>>> my_filter = lambda x: True if x["city"].startswith("new") else False
>>> df = wr.s3.read_parquet(path, dataset=True, partition_filter=my_filter)
"""
session: boto3.Session = _utils.ensure_session(session=boto3_session)
paths: List[str] = _path2list(
path=path,
boto3_session=session,
suffix=path_suffix,
ignore_suffix=_get_path_ignore_suffix(path_ignore_suffix=path_ignore_suffix),
last_modified_begin=last_modified_begin,
last_modified_end=last_modified_end,
ignore_empty=ignore_empty,
s3_additional_kwargs=s3_additional_kwargs,
)
versions: Optional[Dict[str, str]] = (
version_id if isinstance(version_id, dict) else {paths[0]: version_id} if isinstance(version_id, str) else None
)
if path_root is None:
path_root = _get_path_root(path=path, dataset=dataset)
if path_root is not None and partition_filter is not None:
paths = _apply_partition_filter(path_root=path_root, paths=paths, filter_func=partition_filter)
if len(paths) < 1:
raise exceptions.NoFilesFound(f"No files Found on: {path}.")
_logger.debug("paths:\n%s", paths)
args: Dict[str, Any] = {
"columns": columns,
"categories": categories,
"safe": safe,
"map_types": map_types,
"boto3_session": session,
"dataset": dataset,
"path_root": path_root,
"s3_additional_kwargs": s3_additional_kwargs,
"use_threads": use_threads,
"pyarrow_additional_kwargs": pyarrow_additional_kwargs,
}
_logger.debug("args:\n%s", pprint.pformat(args))
if chunked is not False:
return _read_parquet_chunked(
paths=paths,
chunked=chunked,
validate_schema=validate_schema,
ignore_index=ignore_index,
version_ids=versions,
**args,
)
if len(paths) == 1:
return _read_parquet(
path=paths[0], version_id=versions[paths[0]] if isinstance(versions, dict) else None, **args
)
if validate_schema is True:
_validate_schemas_from_files(
paths=paths,
version_ids=versions,
sampling=1.0,
use_threads=True,
boto3_session=boto3_session,
s3_additional_kwargs=s3_additional_kwargs,
)
return _union(
dfs=_read_dfs_from_multiple_paths(
read_func=_read_parquet, paths=paths, version_ids=versions, use_threads=use_threads, kwargs=args
),
ignore_index=ignore_index,
)
