def _read_dfs_from_multiple_paths(
read_func: Callable[..., pd.DataFrame],
paths: List[str],
version_ids: Optional[Dict[str, str]],
use_threads: Union[bool, int],
kwargs: Dict[str, Any],
) -> List[pd.DataFrame]:
cpus = ensure_cpu_count(use_threads)
if cpus < 2:
return [
read_func(
path,
version_id=version_ids.get(path) if version_ids else None,
**kwargs) for path in paths
]
with concurrent.futures.ThreadPoolExecutor(
max_workers=ensure_cpu_count(use_threads)) as executor:
kwargs["boto3_session"] = boto3_to_primitives(kwargs["boto3_session"])
partial_read_func = partial(read_func, **kwargs)
versions = [
version_ids.get(p) if isinstance(version_ids, dict) else None
for p in paths
]
return list(df
for df in executor.map(partial_read_func, paths, versions))
def _read_dfs_from_multiple_paths(
read_func: Callable[..., pd.DataFrame], paths: List[str],
use_threads: Union[bool,
int], kwargs: Dict[str, Any]) -> List[pd.DataFrame]:
cpus = ensure_cpu_count(use_threads)
if cpus < 2:
return [read_func(path, **kwargs) for path in paths]
with concurrent.futures.ThreadPoolExecutor(
max_workers=ensure_cpu_count(use_threads)) as executor:
kwargs["boto3_session"] = boto3_to_primitives(kwargs["boto3_session"])
partial_read_func = partial(read_func, **kwargs)
return list(df for df in executor.map(partial_read_func, paths))
def _read_schemas_from_files(
paths: List[str],
sampling: float,
use_threads: bool,
boto3_session: boto3.Session,
s3_additional_kwargs: Optional[Dict[str, str]],
) -> Tuple[Dict[str, str], ...]:
paths = _utils.list_sampling(lst=paths, sampling=sampling)
schemas: Tuple[Dict[str, str], ...] = tuple()
n_paths: int = len(paths)
if use_threads is False or n_paths == 1:
schemas = tuple(
_read_parquet_metadata_file(
path=p,
boto3_session=boto3_session,
s3_additional_kwargs=s3_additional_kwargs,
use_threads=use_threads) for p in paths)
elif n_paths > 1:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
schemas = tuple(
executor.map(
_read_parquet_metadata_file,
paths,
itertools.repeat(
_utils.boto3_to_primitives(
boto3_session=boto3_session)), # Boto3.Session
itertools.repeat(s3_additional_kwargs),
itertools.repeat(use_threads),
))
_logger.debug("schemas: %s", schemas)
return schemas
def _paginate_stream(args: Dict[str, Any], path: str, use_threads: Union[bool,
int],
boto3_session: Optional[boto3.Session]) -> pd.DataFrame:
obj_size: int = size_objects( # type: ignore
path=[path],
use_threads=False,
boto3_session=boto3_session,
).get(path)
if obj_size is None:
raise exceptions.InvalidArgumentValue(
f"S3 object w/o defined size: {path}")
dfs: List[pd.Dataframe] = []
client_s3: boto3.client = _utils.client(service_name="s3",
session=boto3_session)
if use_threads is False:
dfs = list(
_select_object_content(
args=args,
client_s3=client_s3,
scan_range=scan_range,
) for scan_range in _gen_scan_range(obj_size=obj_size))
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
dfs = list(
executor.map(
_select_object_content,
itertools.repeat(args),
itertools.repeat(client_s3),
_gen_scan_range(obj_size=obj_size),
))
return pd.concat(dfs, ignore_index=True)
def _paginate_stream(
args: Dict[str, Any], path: str, use_threads: Union[bool, int], boto3_session: Optional[boto3.Session]
) -> pd.DataFrame:
obj_size: int = size_objects( # type: ignore
path=[path],
use_threads=False,
boto3_session=boto3_session,
).get(path)
if obj_size is None:
raise exceptions.InvalidArgumentValue(f"S3 object w/o defined size: {path}")
scan_ranges = _gen_scan_range(obj_size=obj_size)
if use_threads is False:
stream_records = list(
_select_object_content(
args=args,
boto3_session=boto3_session,
scan_range=scan_range,
)
for scan_range in scan_ranges
)
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(max_workers=cpus) as executor:
stream_records = list(
executor.map(
_select_object_content,
itertools.repeat(args),
itertools.repeat(boto3_session),
scan_ranges,
)
)
return pd.DataFrame([item for sublist in stream_records for item in sublist]) # Flatten list of lists
def _read_parquet_init(
path: Union[str, List[str]],
filters: Optional[Union[List[Tuple], List[List[Tuple]]]] = None,
categories: List[str] = None,
validate_schema: bool = True,
dataset: bool = False,
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, str]] = None,
) -> pyarrow.parquet.ParquetDataset:
"""Encapsulate all initialization before the use of the pyarrow.parquet.ParquetDataset."""
session: boto3.Session = _utils.ensure_session(session=boto3_session)
if dataset is False:
path_or_paths: Union[str, List[str]] = path2list(path=path,
boto3_session=session)
elif isinstance(path, str):
path_or_paths = path[:-1] if path.endswith("/") else path
else:
path_or_paths = path
_logger.debug("path_or_paths: %s", path_or_paths)
fs: s3fs.S3FileSystem = _utils.get_fs(
session=session, s3_additional_kwargs=s3_additional_kwargs)
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
data: pyarrow.parquet.ParquetDataset = pyarrow.parquet.ParquetDataset(
path_or_paths=path_or_paths,
filesystem=fs,
metadata_nthreads=cpus,
filters=filters,
read_dictionary=categories,
validate_schema=validate_schema,
split_row_groups=False,
use_legacy_dataset=True,
)
return data
def describe_objects(
path: Union[str, List[str]],
wait_time: Optional[Union[int, float]] = None,
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
) -> Dict[str, Dict[str, Any]]:
"""Describe Amazon S3 objects from a received S3 prefix or list of S3 objects paths.
Fetch attributes like ContentLength, DeleteMarker, LastModified, ContentType, etc
The full list of attributes can be explored under the boto3 head_object documentation:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_object
Note
----
In case of `use_threads=True` the number of threads that will be spawned will be get from os.cpu_count().
Parameters
----------
path : Union[str, List[str]]
S3 prefix (e.g. s3://bucket/prefix) or list of S3 objects paths (e.g. [s3://bucket/key0, s3://bucket/key1]).
wait_time : Union[int,float], optional
How much time (seconds) should Wrangler try to reach this objects.
Very useful to overcome eventual consistence issues.
`None` means only a single try will be done.
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
Returns
-------
Dict[str, Dict[str, Any]]
Return a dictionary of objects returned from head_objects where the key is the object path.
The response object can be explored here:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_object
Examples
--------
>>> import awswrangler as wr
>>> descs0 = wr.s3.describe_objects(['s3://bucket/key0', 's3://bucket/key1']) # Describe both objects
>>> descs1 = wr.s3.describe_objects('s3://bucket/prefix') # Describe all objects under the prefix
>>> descs2 = wr.s3.describe_objects('s3://bucket/prefix', wait_time=30) # Overcoming eventual consistence issues
"""
paths: List[str] = path2list(path=path, boto3_session=boto3_session)
if len(paths) < 1:
return {}
client_s3: boto3.client = _utils.client(service_name="s3", session=boto3_session)
resp_list: List[Tuple[str, Dict[str, Any]]]
if use_threads is False:
resp_list = [_describe_object(path=p, wait_time=wait_time, client_s3=client_s3) for p in paths]
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(max_workers=cpus) as executor:
resp_list = list(
executor.map(_describe_object, paths, itertools.repeat(wait_time), itertools.repeat(client_s3))
)
desc_dict: Dict[str, Dict[str, Any]] = dict(resp_list)
return desc_dict
def __init__(self, use_threads: Union[bool, int]):
self._exec: Optional[concurrent.futures.ThreadPoolExecutor]
self._results: List[str] = []
self._cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
if self._cpus > 1:
self._exec = concurrent.futures.ThreadPoolExecutor(max_workers=self._cpus) # pylint: disable=R1732
self._futures: List[Any] = []
else:
self._exec = None
def __init__(self, use_threads: bool):
self._exec: Optional[concurrent.futures.ThreadPoolExecutor]
self._results: List[str] = []
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
if cpus > 1:
self._exec = concurrent.futures.ThreadPoolExecutor(
max_workers=cpus)
self._futures: List[Any] = []
else:
self._exec = None
def _read_parquet(
path: str,
columns: Optional[List[str]],
categories: Optional[List[str]],
safe: bool,
boto3_session: boto3.Session,
dataset: bool,
path_root: Optional[str],
s3_additional_kwargs: Optional[Dict[str, str]],
use_threads: bool,
) -> pd.DataFrame:
if use_threads is False:
table: pa.Table = _read_parquet_file(
path=path,
columns=columns,
categories=categories,
boto3_session=boto3_session,
s3_additional_kwargs=s3_additional_kwargs,
use_threads=use_threads,
)
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
num_row_groups: int = _count_row_groups(
path=path,
categories=categories,
boto3_session=boto3_session,
s3_additional_kwargs=s3_additional_kwargs,
use_threads=use_threads,
)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
tables: Tuple[pa.Table, ...] = tuple(
executor.map(
_read_parquet_row_group,
range(num_row_groups),
itertools.repeat(path),
itertools.repeat(columns),
itertools.repeat(categories),
itertools.repeat(
_utils.boto3_to_primitives(
boto3_session=boto3_session)),
itertools.repeat(s3_additional_kwargs),
itertools.repeat(use_threads),
))
table = pa.lib.concat_tables(tables, promote=False)
_logger.debug("Converting PyArrow Table to Pandas DataFrame...")
return _arrowtable2df(
table=table,
categories=categories,
safe=safe,
use_threads=use_threads,
dataset=dataset,
path=path,
path_root=path_root,
)
def __init__(self, use_threads: Union[bool, int]):
self.closed = False
self._exec: Optional[concurrent.futures.ThreadPoolExecutor]
self._results: List[Dict[str, Union[str, int]]] = []
self._cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
if self._cpus > 1:
self._exec = concurrent.futures.ThreadPoolExecutor(
max_workers=self._cpus)
self._futures: List[Any] = []
else:
self._exec = None
def delete_objects(path: Union[str, List[str]],
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None) -> None:
"""Delete Amazon S3 objects from a received S3 prefix or list of S3 objects paths.
Note
----
In case of `use_threads=True` the number of threads that will be spawned will be get from os.cpu_count().
Parameters
----------
path : Union[str, List[str]]
S3 prefix (e.g. s3://bucket/prefix) or list of S3 objects paths (e.g. [s3://bucket/key0, s3://bucket/key1]).
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
Returns
-------
None
None.
Examples
--------
>>> import awswrangler as wr
>>> wr.s3.delete_objects(['s3://bucket/key0', 's3://bucket/key1']) # Delete both objects
>>> wr.s3.delete_objects('s3://bucket/prefix') # Delete all objects under the received prefix
"""
paths: List[str] = path2list(path=path, boto3_session=boto3_session)
if len(paths) < 1:
return
client_s3: boto3.client = _utils.client(service_name="s3",
session=boto3_session)
buckets: Dict[str, List[str]] = _split_paths_by_bucket(paths=paths)
for bucket, keys in buckets.items():
chunks: List[List[str]] = _utils.chunkify(lst=keys, max_length=1_000)
if use_threads is False:
for chunk in chunks:
_delete_objects(bucket=bucket, keys=chunk, client_s3=client_s3)
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
list(
executor.map(_delete_objects, itertools.repeat(bucket),
chunks, itertools.repeat(client_s3)))
def _read_schemas_from_files(
paths: List[str],
sampling: float,
use_threads: Union[bool, int],
boto3_session: boto3.Session,
s3_additional_kwargs: Optional[Dict[str, str]],
version_ids: Optional[Dict[str, str]] = None,
ignore_null: bool = False,
pyarrow_additional_kwargs: Optional[Dict[str, Any]] = None,
) -> Tuple[Dict[str, str], ...]:
paths = _utils.list_sampling(lst=paths, sampling=sampling)
schemas: Tuple[Optional[Dict[str, str]], ...] = tuple()
n_paths: int = len(paths)
cpus: int = _utils.ensure_cpu_count(use_threads)
if cpus == 1 or n_paths == 1:
schemas = tuple(
_read_parquet_metadata_file(
path=p,
boto3_session=boto3_session,
s3_additional_kwargs=s3_additional_kwargs,
use_threads=use_threads,
version_id=version_ids.get(p) if isinstance(version_ids, dict) else None,
ignore_null=ignore_null,
pyarrow_additional_kwargs=pyarrow_additional_kwargs,
)
for p in paths
)
elif n_paths > 1:
versions = [version_ids.get(p) if isinstance(version_ids, dict) else None for p in paths]
with concurrent.futures.ThreadPoolExecutor(max_workers=cpus) as executor:
schemas = tuple(
executor.map(
_read_parquet_metadata_file,
paths,
itertools.repeat(_utils.boto3_to_primitives(boto3_session=boto3_session)), # Boto3.Session
itertools.repeat(s3_additional_kwargs),
itertools.repeat(use_threads),
versions,
itertools.repeat(ignore_null),
itertools.repeat(pyarrow_additional_kwargs),
)
)
schemas = cast(Tuple[Dict[str, str], ...], tuple(x for x in schemas if x is not None))
_logger.debug("schemas: %s", schemas)
return schemas
def _wait_objects(
waiter_name: str,
paths: List[str],
delay: Optional[Union[int, float]] = None,
max_attempts: Optional[int] = None,
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
) -> None:
delay = 5 if delay is None else delay
max_attempts = 20 if max_attempts is None else max_attempts
_delay: int = int(delay) if isinstance(delay, float) else delay
if len(paths) < 1:
return None
_paths: List[Tuple[str, str]] = [_utils.parse_path(path=p) for p in paths]
if len(_paths) == 1:
_wait_object(
path=_paths[0],
waiter_name=waiter_name,
delay=_delay,
max_attempts=max_attempts,
boto3_session=boto3_session,
)
elif use_threads is False:
for path in _paths:
_wait_object(path=path,
waiter_name=waiter_name,
delay=_delay,
max_attempts=max_attempts,
boto3_session=boto3_session)
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
list(
executor.map(
_wait_object_concurrent,
_paths,
itertools.repeat(waiter_name),
itertools.repeat(_delay),
itertools.repeat(max_attempts),
itertools.repeat(
_utils.boto3_to_primitives(
boto3_session=boto3_session)),
))
return None
def _read_concurrent(
func: Callable[..., pd.DataFrame],
paths: List[str],
ignore_index: Optional[bool],
boto3_session: boto3.Session,
**func_kwargs: Any,
) -> pd.DataFrame:
cpus: int = _utils.ensure_cpu_count(use_threads=True)
with concurrent.futures.ThreadPoolExecutor(max_workers=cpus) as executor:
return _union(
dfs=list(
executor.map(
_caller,
paths,
itertools.repeat(func),
itertools.repeat(_utils.boto3_to_primitives(boto3_session=boto3_session)),
itertools.repeat(func_kwargs),
)
),
ignore_index=ignore_index,
)
def _wait_objects(
waiter_name: str,
paths: List[str],
delay: Optional[Union[int, float]] = None,
max_attempts: Optional[int] = None,
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
) -> None:
delay = 5 if delay is None else delay
max_attempts = 20 if max_attempts is None else max_attempts
_delay: int = int(delay) if isinstance(delay, float) else delay
if len(paths) < 1:
return None
client_s3: boto3.client = _utils.client(service_name="s3",
session=boto3_session)
_paths: List[Tuple[str, str]] = [_utils.parse_path(path=p) for p in paths]
if use_threads is False:
waiter = client_s3.get_waiter(waiter_name)
for bucket, key in _paths:
waiter.wait(Bucket=bucket,
Key=key,
WaiterConfig={
"Delay": _delay,
"MaxAttempts": max_attempts
})
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
list(
executor.map(
_wait_objects_concurrent,
_paths,
itertools.repeat(waiter_name),
itertools.repeat(client_s3),
itertools.repeat(_delay),
itertools.repeat(max_attempts),
))
return None
def _fetch_range_proxy(self, start: int, end: int) -> bytes:
_logger.debug("Fetching: s3://%s/%s - Range: %s-%s", self._bucket, self._key, start, end)
s3_client: boto3.client = _utils.client(service_name="s3", session=self._boto3_session)
boto3_kwargs: Dict[str, Any] = get_botocore_valid_kwargs(
function_name="get_object", s3_additional_kwargs=self._s3_additional_kwargs
)
cpus: int = _utils.ensure_cpu_count(use_threads=self._use_threads)
range_size: int = end - start
if cpus < 2 or range_size < (2 * _MIN_PARALLEL_READ_BLOCK):
return _fetch_range(
range_values=(start, end),
bucket=self._bucket,
key=self._key,
s3_client=s3_client,
boto3_kwargs=boto3_kwargs,
version_id=self._version_id,
)[1]
sizes: Tuple[int, ...] = _utils.get_even_chunks_sizes(
total_size=range_size, chunk_size=_MIN_PARALLEL_READ_BLOCK, upper_bound=False
)
ranges: List[Tuple[int, int]] = []
chunk_start: int = start
for size in sizes:
ranges.append((chunk_start, chunk_start + size))
chunk_start += size
with concurrent.futures.ThreadPoolExecutor(max_workers=cpus) as executor:
return self._merge_range(
ranges=list(
executor.map(
_fetch_range,
ranges,
itertools.repeat(self._bucket),
itertools.repeat(self._key),
itertools.repeat(s3_client),
itertools.repeat(boto3_kwargs),
itertools.repeat(self._version_id),
)
),
)
def to_parquet( # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
df: pd.DataFrame,
path: Optional[str] = None,
index: bool = False,
compression: Optional[str] = "snappy",
pyarrow_additional_kwargs: Optional[Dict[str, Any]] = None,
max_rows_by_file: Optional[int] = None,
use_threads: Union[bool, int] = True,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, Any]] = None,
sanitize_columns: bool = False,
dataset: bool = False,
filename_prefix: Optional[str] = None,
partition_cols: Optional[List[str]] = None,
bucketing_info: Optional[Tuple[List[str], int]] = None,
concurrent_partitioning: bool = False,
mode: Optional[str] = None,
catalog_versioning: bool = False,
schema_evolution: bool = True,
database: Optional[str] = None,
table: Optional[str] = None,
table_type: Optional[str] = None,
transaction_id: Optional[str] = None,
dtype: Optional[Dict[str, str]] = None,
description: Optional[str] = None,
parameters: Optional[Dict[str, str]] = None,
columns_comments: Optional[Dict[str, str]] = None,
regular_partitions: bool = True,
projection_enabled: bool = False,
projection_types: Optional[Dict[str, str]] = None,
projection_ranges: Optional[Dict[str, str]] = None,
projection_values: Optional[Dict[str, str]] = None,
projection_intervals: Optional[Dict[str, str]] = None,
projection_digits: Optional[Dict[str, str]] = None,
catalog_id: Optional[str] = None,
) -> Dict[str, Union[List[str], Dict[str, List[str]]]]:
"""Write Parquet file or dataset on Amazon S3.
The concept of Dataset goes beyond the simple idea of ordinary files and enable more
complex features like partitioning and catalog integration (Amazon Athena/AWS Glue Catalog).
Note
----
This operation may mutate the original pandas dataframe in-place. To avoid this behaviour
please pass in a deep copy instead (i.e. `df.copy()`)
Note
----
If `database` and `table` arguments are passed, the table name and all column names
will be automatically sanitized using `wr.catalog.sanitize_table_name` and `wr.catalog.sanitize_column_name`.
Please, pass `sanitize_columns=True` to enforce this behaviour always.
Note
----
On `append` mode, the `parameters` will be upsert on an existing table.
Note
----
In case of `use_threads=True` the number of threads
that will be spawned will be gotten from os.cpu_count().
Parameters
----------
df: pandas.DataFrame
Pandas DataFrame https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
path : str, optional
S3 path (for file e.g. ``s3://bucket/prefix/filename.parquet``) (for dataset e.g. ``s3://bucket/prefix``).
Required if dataset=False or when dataset=True and creating a new dataset
index : bool
True to store the DataFrame index in file, otherwise False to ignore it.
compression: str, optional
Compression style (``None``, ``snappy``, ``gzip``).
pyarrow_additional_kwargs : Optional[Dict[str, Any]]
Additional parameters forwarded to pyarrow.
e.g. pyarrow_additional_kwargs={'coerce_timestamps': 'ns', 'use_deprecated_int96_timestamps': False,
'allow_truncated_timestamps'=False}
max_rows_by_file : int
Max number of rows in each file.
Default is None i.e. dont split the files.
(e.g. 33554432, 268435456)
use_threads : bool, int
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
If integer is provided, specified number is used.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
s3_additional_kwargs : Optional[Dict[str, Any]]
Forwarded to botocore requests.
e.g. s3_additional_kwargs={'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'}
sanitize_columns : bool
True to sanitize columns names (using `wr.catalog.sanitize_table_name` and `wr.catalog.sanitize_column_name`)
or False to keep it as is.
True value behaviour is enforced if `database` and `table` arguments are passed.
dataset : bool
If True store a parquet dataset instead of a ordinary file(s)
If True, enable all follow arguments:
partition_cols, mode, database, table, description, parameters, columns_comments, concurrent_partitioning,
catalog_versioning, projection_enabled, projection_types, projection_ranges, projection_values,
projection_intervals, projection_digits, catalog_id, schema_evolution.
filename_prefix: str, optional
If dataset=True, add a filename prefix to the output files.
partition_cols: List[str], optional
List of column names that will be used to create partitions. Only takes effect if dataset=True.
bucketing_info: Tuple[List[str], int], optional
Tuple consisting of the column names used for bucketing as the first element and the number of buckets as the
second element.
Only `str`, `int` and `bool` are supported as column data types for bucketing.
concurrent_partitioning: bool
If True will increase the parallelism level during the partitions writing. It will decrease the
writing time and increase the memory usage.
https://aws-data-wrangler.readthedocs.io/en/2.13.0/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html
mode: str, optional
``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True.
For details check the related tutorial:
https://aws-data-wrangler.readthedocs.io/en/2.13.0/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet
catalog_versioning : bool
If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it.
schema_evolution : bool
If True allows schema evolution (new or missing columns), otherwise a exception will be raised. True by default.
(Only considered if dataset=True and mode in ("append", "overwrite_partitions"))
Related tutorial:
https://aws-data-wrangler.readthedocs.io/en/2.13.0/tutorials/014%20-%20Schema%20Evolution.html
database : str, optional
Glue/Athena catalog: Database name.
table : str, optional
Glue/Athena catalog: Table name.
table_type: str, optional
The type of the Glue Table. Set to EXTERNAL_TABLE if None.
transaction_id: str, optional
The ID of the transaction when writing to a Governed Table.
dtype : Dict[str, str], optional
Dictionary of columns names and Athena/Glue types to be casted.
Useful when you have columns with undetermined or mixed data types.
(e.g. {'col name': 'bigint', 'col2 name': 'int'})
description : str, optional
Glue/Athena catalog: Table description
parameters : Dict[str, str], optional
Glue/Athena catalog: Key/value pairs to tag the table.
columns_comments : Dict[str, str], optional
Glue/Athena catalog:
Columns names and the related comments (e.g. {'col0': 'Column 0.', 'col1': 'Column 1.', 'col2': 'Partition.'}).
regular_partitions : bool
Create regular partitions (Non projected partitions) on Glue Catalog.
Disable when you will work only with Partition Projection.
Keep enabled even when working with projections is useful to keep
Redshift Spectrum working with the regular partitions.
projection_enabled : bool
Enable Partition Projection on Athena (https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html)
projection_types : Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections types.
Valid types: "enum", "integer", "date", "injected"
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'enum', 'col2_name': 'integer'})
projection_ranges: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections ranges.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '0,10', 'col2_name': '-1,8675309'})
projection_values: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections values.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'A,B,Unknown', 'col2_name': 'foo,boo,bar'})
projection_intervals: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections intervals.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '5'})
projection_digits: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections digits.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '2'})
catalog_id : str, optional
The ID of the Data Catalog from which to retrieve Databases.
If none is provided, the AWS account ID is used by default.
Returns
-------
Dict[str, Union[List[str], Dict[str, List[str]]]]
Dictionary with:
'paths': List of all stored files paths on S3.
'partitions_values': Dictionary of partitions added with keys as S3 path locations
and values as a list of partitions values as str.
Examples
--------
Writing single file
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.parquet',
... )
{
'paths': ['s3://bucket/prefix/my_file.parquet'],
'partitions_values': {}
}
Writing single file encrypted with a KMS key
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.parquet',
... s3_additional_kwargs={
... 'ServerSideEncryption': 'aws:kms',
... 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'
... }
... )
{
'paths': ['s3://bucket/prefix/my_file.parquet'],
'partitions_values': {}
}
Writing partitioned dataset
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2']
... )
{
'paths': ['s3://.../col2=A/x.parquet', 's3://.../col2=B/y.parquet'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing bucketed dataset
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... bucketing_info=(["col2"], 2)
... )
{
'paths': ['s3://.../x_bucket-00000.csv', 's3://.../col2=B/x_bucket-00001.csv'],
'partitions_values: {}
}
Writing dataset to S3 with metadata on Athena/Glue Catalog.
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2'],
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... )
{
'paths': ['s3://.../col2=A/x.parquet', 's3://.../col2=B/y.parquet'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset to Glue governed table
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B'],
... 'col3': [None, None, None]
... }),
... dataset=True,
... mode='append',
... database='default', # Athena/Glue database
... table='my_table', # Athena/Glue table
... table_type='GOVERNED',
... transaction_id="xxx",
... )
{
'paths': ['s3://.../x.parquet'],
'partitions_values: {}
}
Writing dataset casting empty column data type
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B'],
... 'col3': [None, None, None]
... }),
... path='s3://bucket/prefix',
... dataset=True,
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... dtype={'col3': 'date'}
... )
{
'paths': ['s3://.../x.parquet'],
'partitions_values: {}
}
"""
_validate_args(
df=df,
table=table,
database=database,
dataset=dataset,
path=path,
partition_cols=partition_cols,
bucketing_info=bucketing_info,
mode=mode,
description=description,
parameters=parameters,
columns_comments=columns_comments,
)
# Evaluating compression
if _COMPRESSION_2_EXT.get(compression, None) is None:
raise exceptions.InvalidCompression(f"{compression} is invalid, please use None, 'snappy' or 'gzip'.")
compression_ext: str = _COMPRESSION_2_EXT[compression]
# Initializing defaults
partition_cols = partition_cols if partition_cols else []
dtype = dtype if dtype else {}
partitions_values: Dict[str, List[str]] = {}
mode = "append" if mode is None else mode
commit_trans: bool = False
if transaction_id:
table_type = "GOVERNED"
filename_prefix = filename_prefix + uuid.uuid4().hex if filename_prefix else uuid.uuid4().hex
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
session: boto3.Session = _utils.ensure_session(session=boto3_session)
# Sanitize table to respect Athena's standards
if (sanitize_columns is True) or (database is not None and table is not None):
df, dtype, partition_cols = _sanitize(df=df, dtype=dtype, partition_cols=partition_cols)
# Evaluating dtype
catalog_table_input: Optional[Dict[str, Any]] = None
if database is not None and table is not None:
catalog_table_input = catalog._get_table_input( # pylint: disable=protected-access
database=database, table=table, boto3_session=session, transaction_id=transaction_id, catalog_id=catalog_id
)
catalog_path: Optional[str] = None
if catalog_table_input:
table_type = catalog_table_input["TableType"]
catalog_path = catalog_table_input["StorageDescriptor"]["Location"]
if path is None:
if catalog_path:
path = catalog_path
else:
raise exceptions.InvalidArgumentValue(
"Glue table does not exist in the catalog. Please pass the `path` argument to create it."
)
elif path and catalog_path:
if path.rstrip("/") != catalog_path.rstrip("/"):
raise exceptions.InvalidArgumentValue(
f"The specified path: {path}, does not match the existing Glue catalog table path: {catalog_path}"
)
if (table_type == "GOVERNED") and (not transaction_id):
_logger.debug("`transaction_id` not specified for GOVERNED table, starting transaction")
transaction_id = lakeformation.start_transaction(read_only=False, boto3_session=boto3_session)
commit_trans = True
df = _apply_dtype(df=df, dtype=dtype, catalog_table_input=catalog_table_input, mode=mode)
schema: pa.Schema = _data_types.pyarrow_schema_from_pandas(
df=df, index=index, ignore_cols=partition_cols, dtype=dtype
)
_logger.debug("schema: \n%s", schema)
if dataset is False:
paths = _to_parquet(
df=df,
path=path,
schema=schema,
index=index,
cpus=cpus,
compression=compression,
compression_ext=compression_ext,
pyarrow_additional_kwargs=pyarrow_additional_kwargs,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
dtype=dtype,
max_rows_by_file=max_rows_by_file,
use_threads=use_threads,
)
else:
columns_types: Dict[str, str] = {}
partitions_types: Dict[str, str] = {}
if (database is not None) and (table is not None):
columns_types, partitions_types = _data_types.athena_types_from_pandas_partitioned(
df=df, index=index, partition_cols=partition_cols, dtype=dtype
)
if schema_evolution is False:
_utils.check_schema_changes(columns_types=columns_types, table_input=catalog_table_input, mode=mode)
if (catalog_table_input is None) and (table_type == "GOVERNED"):
catalog._create_parquet_table( # pylint: disable=protected-access
database=database,
table=table,
path=path, # type: ignore
columns_types=columns_types,
table_type=table_type,
partitions_types=partitions_types,
bucketing_info=bucketing_info,
compression=compression,
description=description,
parameters=parameters,
columns_comments=columns_comments,
boto3_session=session,
mode=mode,
transaction_id=transaction_id,
catalog_versioning=catalog_versioning,
projection_enabled=projection_enabled,
projection_types=projection_types,
projection_ranges=projection_ranges,
projection_values=projection_values,
projection_intervals=projection_intervals,
projection_digits=projection_digits,
projection_storage_location_template=None,
catalog_id=catalog_id,
catalog_table_input=catalog_table_input,
)
catalog_table_input = catalog._get_table_input( # pylint: disable=protected-access
database=database,
table=table,
boto3_session=session,
transaction_id=transaction_id,
catalog_id=catalog_id,
)
paths, partitions_values = _to_dataset(
func=_to_parquet,
concurrent_partitioning=concurrent_partitioning,
df=df,
path_root=path, # type: ignore
filename_prefix=filename_prefix,
index=index,
compression=compression,
compression_ext=compression_ext,
catalog_id=catalog_id,
database=database,
table=table,
table_type=table_type,
transaction_id=transaction_id,
pyarrow_additional_kwargs=pyarrow_additional_kwargs,
cpus=cpus,
use_threads=use_threads,
partition_cols=partition_cols,
partitions_types=partitions_types,
bucketing_info=bucketing_info,
dtype=dtype,
mode=mode,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
schema=schema,
max_rows_by_file=max_rows_by_file,
)
if (database is not None) and (table is not None):
try:
catalog._create_parquet_table( # pylint: disable=protected-access
database=database,
table=table,
path=path, # type: ignore
columns_types=columns_types,
table_type=table_type,
partitions_types=partitions_types,
bucketing_info=bucketing_info,
compression=compression,
description=description,
parameters=parameters,
columns_comments=columns_comments,
boto3_session=session,
mode=mode,
transaction_id=transaction_id,
catalog_versioning=catalog_versioning,
projection_enabled=projection_enabled,
projection_types=projection_types,
projection_ranges=projection_ranges,
projection_values=projection_values,
projection_intervals=projection_intervals,
projection_digits=projection_digits,
projection_storage_location_template=None,
catalog_id=catalog_id,
catalog_table_input=catalog_table_input,
)
if partitions_values and (regular_partitions is True) and (table_type != "GOVERNED"):
_logger.debug("partitions_values:\n%s", partitions_values)
catalog.add_parquet_partitions(
database=database,
table=table,
partitions_values=partitions_values,
bucketing_info=bucketing_info,
compression=compression,
boto3_session=session,
catalog_id=catalog_id,
columns_types=columns_types,
)
if commit_trans:
lakeformation.commit_transaction(
transaction_id=transaction_id, boto3_session=boto3_session # type: ignore
)
except Exception:
_logger.debug("Catalog write failed, cleaning up S3 (paths: %s).", paths)
delete_objects(
path=paths,
use_threads=use_threads,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
)
raise
return {"paths": paths, "partitions_values": partitions_values}
def _resolve_sql_query(
query_id: str,
categories: Optional[List[str]],
safe: bool,
map_types: bool,
use_threads: bool,
boto3_session: boto3.Session,
) -> pd.DataFrame:
client_lakeformation: boto3.client = _utils.client(
service_name="lakeformation", session=boto3_session)
wait_query(query_id=query_id, boto3_session=boto3_session)
# The LF Query Engine distributes the load across workers
# Retrieve the tokens and their associated work units until NextToken is ''
# One Token can span multiple work units
# PageSize determines the size of the "Units" array in each call
scan_kwargs: Dict[str, Union[str, int]] = {
"QueryId": query_id,
"PageSize": 10
}
next_token: str = "init_token" # Dummy token
token_work_units: List[Tuple[str, int]] = []
while next_token:
response = client_lakeformation.get_work_units(**scan_kwargs)
token_work_units.extend( # [(Token0, WorkUnitId0), (Token0, WorkUnitId1), (Token1, WorkUnitId2) ... ]
[
(unit["WorkUnitToken"], unit_id)
for unit in response["WorkUnitRanges"] for unit_id in range(
unit["WorkUnitIdMin"], unit["WorkUnitIdMax"] +
1) # Max is inclusive
])
next_token = response.get("NextToken", None)
scan_kwargs["NextToken"] = next_token
tables: List[Table] = []
if use_threads is False:
tables = list(
_get_work_unit_results(
query_id=query_id,
token_work_unit=token_work_unit,
client_lakeformation=client_lakeformation,
) for token_work_unit in token_work_units)
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
tables = list(
executor.map(
_get_work_unit_results,
itertools.repeat(query_id),
token_work_units,
itertools.repeat(client_lakeformation),
))
table = concat_tables(tables)
args = {
"use_threads":
use_threads,
"split_blocks":
True,
"self_destruct":
True,
"integer_object_nulls":
False,
"date_as_object":
True,
"ignore_metadata":
True,
"strings_to_categorical":
False,
"categories":
categories,
"safe":
safe,
"types_mapper":
_data_types.pyarrow2pandas_extension if map_types else None,
}
return _utils.ensure_df_is_mutable(df=table.to_pandas(**args))
def to_parquet( # pylint: disable=too-many-arguments,too-many-locals
df: pd.DataFrame,
path: str,
index: bool = False,
compression: Optional[str] = "snappy",
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, str]] = None,
sanitize_columns: bool = False,
dataset: bool = False,
partition_cols: Optional[List[str]] = None,
mode: Optional[str] = None,
catalog_versioning: bool = False,
database: Optional[str] = None,
table: Optional[str] = None,
dtype: Optional[Dict[str, str]] = None,
description: Optional[str] = None,
parameters: Optional[Dict[str, str]] = None,
columns_comments: Optional[Dict[str, str]] = None,
regular_partitions: bool = True,
projection_enabled: bool = False,
projection_types: Optional[Dict[str, str]] = None,
projection_ranges: Optional[Dict[str, str]] = None,
projection_values: Optional[Dict[str, str]] = None,
projection_intervals: Optional[Dict[str, str]] = None,
projection_digits: Optional[Dict[str, str]] = None,
) -> Dict[str, Union[List[str], Dict[str, List[str]]]]:
"""Write Parquet file or dataset on Amazon S3.
The concept of Dataset goes beyond the simple idea of files and enable more
complex features like partitioning, casting and catalog integration (Amazon Athena/AWS Glue Catalog).
Note
----
If `dataset=True` The table name and all column names will be automatically sanitized using
`wr.catalog.sanitize_table_name` and `wr.catalog.sanitize_column_name`.
Please, pass `sanitize_columns=True` to force the same behaviour for `dataset=False`.
Note
----
On `append` mode, the `parameters` will be upsert on an existing table.
Note
----
In case of `use_threads=True` the number of threads that will be spawned will be get from os.cpu_count().
Parameters
----------
df: pandas.DataFrame
Pandas DataFrame https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
path : str
S3 path (for file e.g. ``s3://bucket/prefix/filename.parquet``) (for dataset e.g. ``s3://bucket/prefix``).
index : bool
True to store the DataFrame index in file, otherwise False to ignore it.
compression: str, optional
Compression style (``None``, ``snappy``, ``gzip``).
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
s3_additional_kwargs:
Forward to s3fs, useful for server side encryption
https://s3fs.readthedocs.io/en/latest/#serverside-encryption
sanitize_columns : bool
True to sanitize columns names or False to keep it as is.
True value is forced if `dataset=True`.
dataset : bool
If True store a parquet dataset instead of a single file.
If True, enable all follow arguments:
partition_cols, mode, database, table, description, parameters, columns_comments, .
partition_cols: List[str], optional
List of column names that will be used to create partitions. Only takes effect if dataset=True.
mode: str, optional
``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True.
catalog_versioning : bool
If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it.
database : str, optional
Glue/Athena catalog: Database name.
table : str, optional
Glue/Athena catalog: Table name.
dtype : Dict[str, str], optional
Dictionary of columns names and Athena/Glue types to be casted.
Useful when you have columns with undetermined or mixed data types.
(e.g. {'col name': 'bigint', 'col2 name': 'int'})
description : str, optional
Glue/Athena catalog: Table description
parameters : Dict[str, str], optional
Glue/Athena catalog: Key/value pairs to tag the table.
columns_comments : Dict[str, str], optional
Glue/Athena catalog:
Columns names and the related comments (e.g. {'col0': 'Column 0.', 'col1': 'Column 1.', 'col2': 'Partition.'}).
regular_partitions : bool
Create regular partitions (Non projected partitions) on Glue Catalog.
Disable when you will work only with Partition Projection.
Keep enabled even when working with projections is useful to keep
Redshift Spectrum working with the regular partitions.
projection_enabled : bool
Enable Partition Projection on Athena (https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html)
projection_types : Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections types.
Valid types: "enum", "integer", "date", "injected"
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'enum', 'col2_name': 'integer'})
projection_ranges: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections ranges.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '0,10', 'col2_name': '-1,8675309'})
projection_values: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections values.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'A,B,Unknown', 'col2_name': 'foo,boo,bar'})
projection_intervals: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections intervals.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '5'})
projection_digits: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections digits.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '2'})
Returns
-------
Dict[str, Union[List[str], Dict[str, List[str]]]]
Dictionary with:
'paths': List of all stored files paths on S3.
'partitions_values': Dictionary of partitions added with keys as S3 path locations
and values as a list of partitions values as str.
Examples
--------
Writing single file
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.parquet',
... )
{
'paths': ['s3://bucket/prefix/my_file.parquet'],
'partitions_values': {}
}
Writing single file encrypted with a KMS key
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.parquet',
... s3_additional_kwargs={
... 'ServerSideEncryption': 'aws:kms',
... 'SSEKMSKeyId': 'YOUR_KMY_KEY_ARN'
... }
... )
{
'paths': ['s3://bucket/prefix/my_file.parquet'],
'partitions_values': {}
}
Writing partitioned dataset
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2']
... )
{
'paths': ['s3://.../col2=A/x.parquet', 's3://.../col2=B/y.parquet'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset to S3 with metadata on Athena/Glue Catalog.
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2'],
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... )
{
'paths': ['s3://.../col2=A/x.parquet', 's3://.../col2=B/y.parquet'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset casting empty column data type
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B'],
... 'col3': [None, None, None]
... }),
... path='s3://bucket/prefix',
... dataset=True,
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... dtype={'col3': 'date'}
... )
{
'paths': ['s3://.../x.parquet'],
'partitions_values: {}
}
"""
if (database is None) ^ (table is None):
raise exceptions.InvalidArgumentCombination(
"Please pass database and table arguments to be able to store the metadata into the Athena/Glue Catalog."
)
if df.empty is True:
raise exceptions.EmptyDataFrame()
partition_cols = partition_cols if partition_cols else []
dtype = dtype if dtype else {}
partitions_values: Dict[str, List[str]] = {}
# Sanitize table to respect Athena's standards
if (sanitize_columns is True) or (dataset is True):
df = catalog.sanitize_dataframe_columns_names(df=df)
partition_cols = [catalog.sanitize_column_name(p) for p in partition_cols]
dtype = {catalog.sanitize_column_name(k): v.lower() for k, v in dtype.items()}
catalog.drop_duplicated_columns(df=df)
session: boto3.Session = _utils.ensure_session(session=boto3_session)
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
fs: s3fs.S3FileSystem = _utils.get_fs(session=session, s3_additional_kwargs=s3_additional_kwargs)
compression_ext: Optional[str] = _COMPRESSION_2_EXT.get(compression, None)
if compression_ext is None:
raise exceptions.InvalidCompression(f"{compression} is invalid, please use None, snappy or gzip.")
if dataset is False:
if path.endswith("/"): # pragma: no cover
raise exceptions.InvalidArgumentValue(
"If <dataset=False>, the argument <path> should be a object path, not a directory."
)
if partition_cols:
raise exceptions.InvalidArgumentCombination("Please, pass dataset=True to be able to use partition_cols.")
if mode is not None:
raise exceptions.InvalidArgumentCombination("Please pass dataset=True to be able to use mode.")
if any(arg is not None for arg in (database, table, description, parameters)):
raise exceptions.InvalidArgumentCombination(
"Please pass dataset=True to be able to use any one of these "
"arguments: database, table, description, parameters, "
"columns_comments."
)
df = _data_types.cast_pandas_with_athena_types(df=df, dtype=dtype)
schema: pa.Schema = _data_types.pyarrow_schema_from_pandas(
df=df, index=index, ignore_cols=partition_cols, dtype=dtype
)
_logger.debug("schema: \n%s", schema)
paths = [
_to_parquet_file(
df=df, path=path, schema=schema, index=index, compression=compression, cpus=cpus, fs=fs, dtype=dtype
)
]
else:
mode = "append" if mode is None else mode
if (
(mode in ("append", "overwrite_partitions")) and (database is not None) and (table is not None)
): # Fetching Catalog Types
catalog_types: Optional[Dict[str, str]] = catalog.get_table_types(
database=database, table=table, boto3_session=session
)
if catalog_types is not None:
for k, v in catalog_types.items():
dtype[k] = v
paths, partitions_values = _to_parquet_dataset(
df=df,
path=path,
index=index,
compression=compression,
compression_ext=compression_ext,
cpus=cpus,
fs=fs,
use_threads=use_threads,
partition_cols=partition_cols,
dtype=dtype,
mode=mode,
boto3_session=session,
)
if (database is not None) and (table is not None):
columns_types, partitions_types = _data_types.athena_types_from_pandas_partitioned(
df=df, index=index, partition_cols=partition_cols, dtype=dtype
)
catalog.create_parquet_table(
database=database,
table=table,
path=path,
columns_types=columns_types,
partitions_types=partitions_types,
compression=compression,
description=description,
parameters=parameters,
columns_comments=columns_comments,
boto3_session=session,
mode=mode,
catalog_versioning=catalog_versioning,
projection_enabled=projection_enabled,
projection_types=projection_types,
projection_ranges=projection_ranges,
projection_values=projection_values,
projection_intervals=projection_intervals,
projection_digits=projection_digits,
)
if partitions_values and (regular_partitions is True):
_logger.debug("partitions_values:\n%s", partitions_values)
catalog.add_parquet_partitions(
database=database,
table=table,
partitions_values=partitions_values,
compression=compression,
boto3_session=session,
)
return {"paths": paths, "partitions_values": partitions_values}
def to_parquet( # pylint: disable=too-many-arguments,too-many-locals
df: pd.DataFrame,
path: str,
index: bool = False,
compression: Optional[str] = "snappy",
max_rows_by_file: Optional[int] = None,
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, str]] = None,
sanitize_columns: bool = False,
dataset: bool = False,
partition_cols: Optional[List[str]] = None,
concurrent_partitioning: bool = False,
mode: Optional[str] = None,
catalog_versioning: bool = False,
database: Optional[str] = None,
table: Optional[str] = None,
dtype: Optional[Dict[str, str]] = None,
description: Optional[str] = None,
parameters: Optional[Dict[str, str]] = None,
columns_comments: Optional[Dict[str, str]] = None,
regular_partitions: bool = True,
projection_enabled: bool = False,
projection_types: Optional[Dict[str, str]] = None,
projection_ranges: Optional[Dict[str, str]] = None,
projection_values: Optional[Dict[str, str]] = None,
projection_intervals: Optional[Dict[str, str]] = None,
projection_digits: Optional[Dict[str, str]] = None,
catalog_id: Optional[str] = None,
) -> Dict[str, Union[List[str], Dict[str, List[str]]]]:
"""Write Parquet file or dataset on Amazon S3.
The concept of Dataset goes beyond the simple idea of files and enable more
complex features like partitioning, casting and catalog integration (Amazon Athena/AWS Glue Catalog).
Note
----
If `dataset=True` The table name and all column names will be automatically sanitized using
`wr.catalog.sanitize_table_name` and `wr.catalog.sanitize_column_name`.
Please, pass `sanitize_columns=True` to force the same behaviour for `dataset=False`.
Note
----
On `append` mode, the `parameters` will be upsert on an existing table.
Note
----
In case of `use_threads=True` the number of threads
that will be spawned will be gotten from os.cpu_count().
Parameters
----------
df: pandas.DataFrame
Pandas DataFrame https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
path : str
S3 path (for file e.g. ``s3://bucket/prefix/filename.parquet``) (for dataset e.g. ``s3://bucket/prefix``).
index : bool
True to store the DataFrame index in file, otherwise False to ignore it.
compression: str, optional
Compression style (``None``, ``snappy``, ``gzip``).
max_rows_by_file : int
Max number of rows in each file.
Default is None i.e. dont split the files.
(e.g. 33554432, 268435456)
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
s3_additional_kwargs:
Forward to s3fs, useful for server side encryption
https://s3fs.readthedocs.io/en/latest/#serverside-encryption
sanitize_columns : bool
True to sanitize columns names or False to keep it as is.
True value is forced if `dataset=True`.
dataset : bool
If True store a parquet dataset instead of a single file.
If True, enable all follow arguments:
partition_cols, mode, database, table, description, parameters, columns_comments, .
partition_cols: List[str], optional
List of column names that will be used to create partitions. Only takes effect if dataset=True.
concurrent_partitioning: bool
If True will increase the parallelism level during the partitions writing. It will decrease the
writing time and increase the memory usage.
https://github.com/awslabs/aws-data-wrangler/blob/master/tutorials/022%20-%20Writing%20Partitions%20Concurrently.ipynb
mode: str, optional
``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True.
For details check the related tutorial:
https://aws-data-wrangler.readthedocs.io/en/latest/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet
catalog_versioning : bool
If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it.
database : str, optional
Glue/Athena catalog: Database name.
table : str, optional
Glue/Athena catalog: Table name.
dtype : Dict[str, str], optional
Dictionary of columns names and Athena/Glue types to be casted.
Useful when you have columns with undetermined or mixed data types.
(e.g. {'col name': 'bigint', 'col2 name': 'int'})
description : str, optional
Glue/Athena catalog: Table description
parameters : Dict[str, str], optional
Glue/Athena catalog: Key/value pairs to tag the table.
columns_comments : Dict[str, str], optional
Glue/Athena catalog:
Columns names and the related comments (e.g. {'col0': 'Column 0.', 'col1': 'Column 1.', 'col2': 'Partition.'}).
regular_partitions : bool
Create regular partitions (Non projected partitions) on Glue Catalog.
Disable when you will work only with Partition Projection.
Keep enabled even when working with projections is useful to keep
Redshift Spectrum working with the regular partitions.
projection_enabled : bool
Enable Partition Projection on Athena (https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html)
projection_types : Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections types.
Valid types: "enum", "integer", "date", "injected"
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'enum', 'col2_name': 'integer'})
projection_ranges: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections ranges.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '0,10', 'col2_name': '-1,8675309'})
projection_values: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections values.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'A,B,Unknown', 'col2_name': 'foo,boo,bar'})
projection_intervals: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections intervals.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '5'})
projection_digits: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections digits.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '2'})
catalog_id : str, optional
The ID of the Data Catalog from which to retrieve Databases.
If none is provided, the AWS account ID is used by default.
Returns
-------
Dict[str, Union[List[str], Dict[str, List[str]]]]
Dictionary with:
'paths': List of all stored files paths on S3.
'partitions_values': Dictionary of partitions added with keys as S3 path locations
and values as a list of partitions values as str.
Examples
--------
Writing single file
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.parquet',
... )
{
'paths': ['s3://bucket/prefix/my_file.parquet'],
'partitions_values': {}
}
Writing single file encrypted with a KMS key
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.parquet',
... s3_additional_kwargs={
... 'ServerSideEncryption': 'aws:kms',
... 'SSEKMSKeyId': 'YOUR_KMY_KEY_ARN'
... }
... )
{
'paths': ['s3://bucket/prefix/my_file.parquet'],
'partitions_values': {}
}
Writing partitioned dataset
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2']
... )
{
'paths': ['s3://.../col2=A/x.parquet', 's3://.../col2=B/y.parquet'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset to S3 with metadata on Athena/Glue Catalog.
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2'],
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... )
{
'paths': ['s3://.../col2=A/x.parquet', 's3://.../col2=B/y.parquet'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset casting empty column data type
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B'],
... 'col3': [None, None, None]
... }),
... path='s3://bucket/prefix',
... dataset=True,
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... dtype={'col3': 'date'}
... )
{
'paths': ['s3://.../x.parquet'],
'partitions_values: {}
}
"""
_validate_args(
df=df,
table=table,
dataset=dataset,
path=path,
partition_cols=partition_cols,
mode=mode,
description=description,
parameters=parameters,
columns_comments=columns_comments,
)
# Evaluating compression
if _COMPRESSION_2_EXT.get(compression, None) is None:
raise exceptions.InvalidCompression(
f"{compression} is invalid, please use None, 'snappy' or 'gzip'.")
compression_ext: str = _COMPRESSION_2_EXT[compression]
# Initializing defaults
partition_cols = partition_cols if partition_cols else []
dtype = dtype if dtype else {}
partitions_values: Dict[str, List[str]] = {}
mode = "append" if mode is None else mode
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
session: boto3.Session = _utils.ensure_session(session=boto3_session)
# Sanitize table to respect Athena's standards
if (sanitize_columns is True) or (dataset is True):
df, dtype, partition_cols = _sanitize(df=df,
dtype=dtype,
partition_cols=partition_cols)
# Evaluating dtype
catalog_table_input: Optional[Dict[str, Any]] = None
if database is not None and table is not None:
catalog_table_input = catalog._get_table_input( # pylint: disable=protected-access
database=database,
table=table,
boto3_session=session,
catalog_id=catalog_id)
df = _apply_dtype(df=df,
dtype=dtype,
catalog_table_input=catalog_table_input,
mode=mode)
schema: pa.Schema = _data_types.pyarrow_schema_from_pandas(
df=df, index=index, ignore_cols=partition_cols, dtype=dtype)
_logger.debug("schema: \n%s", schema)
if dataset is False:
paths = _to_parquet(
df=df,
path=path,
schema=schema,
index=index,
cpus=cpus,
compression=compression,
compression_ext=compression_ext,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
dtype=dtype,
max_rows_by_file=max_rows_by_file,
)
else:
paths, partitions_values = _to_dataset(
func=_to_parquet,
concurrent_partitioning=concurrent_partitioning,
df=df,
path_root=path,
index=index,
compression=compression,
compression_ext=compression_ext,
cpus=cpus,
use_threads=use_threads,
partition_cols=partition_cols,
dtype=dtype,
mode=mode,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
schema=schema,
max_rows_by_file=max_rows_by_file,
)
if (database is not None) and (table is not None):
columns_types, partitions_types = _data_types.athena_types_from_pandas_partitioned(
df=df, index=index, partition_cols=partition_cols, dtype=dtype)
catalog._create_parquet_table( # pylint: disable=protected-access
database=database,
table=table,
path=path,
columns_types=columns_types,
partitions_types=partitions_types,
compression=compression,
description=description,
parameters=parameters,
columns_comments=columns_comments,
boto3_session=session,
mode=mode,
catalog_versioning=catalog_versioning,
projection_enabled=projection_enabled,
projection_types=projection_types,
projection_ranges=projection_ranges,
projection_values=projection_values,
projection_intervals=projection_intervals,
projection_digits=projection_digits,
catalog_id=catalog_id,
catalog_table_input=catalog_table_input,
)
if partitions_values and (regular_partitions is True):
_logger.debug("partitions_values:\n%s", partitions_values)
catalog.add_parquet_partitions(
database=database,
table=table,
partitions_values=partitions_values,
compression=compression,
boto3_session=session,
)
return {"paths": paths, "partitions_values": partitions_values}
def describe_objects(
path: Union[str, List[str]],
use_threads: bool = True,
last_modified_begin: Optional[datetime.datetime] = None,
last_modified_end: Optional[datetime.datetime] = None,
boto3_session: Optional[boto3.Session] = None,
) -> Dict[str, Dict[str, Any]]:
"""Describe Amazon S3 objects from a received S3 prefix or list of S3 objects paths.
Fetch attributes like ContentLength, DeleteMarker, last_modified, ContentType, etc
The full list of attributes can be explored under the boto3 head_object documentation:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_object
This function accepts Unix shell-style wildcards in the path argument.
* (matches everything), ? (matches any single character),
[seq] (matches any character in seq), [!seq] (matches any character not in seq).
Note
----
In case of `use_threads=True` the number of threads
that will be spawned will be gotten from os.cpu_count().
Note
----
The filter by last_modified begin last_modified end is applied after list all S3 files
Parameters
----------
path : Union[str, List[str]]
S3 prefix (accepts Unix shell-style wildcards)
(e.g. s3://bucket/prefix) or list of S3 objects paths (e.g. [s3://bucket/key0, s3://bucket/key1]).
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
last_modified_begin
Filter the s3 files by the Last modified date of the object.
The filter is applied only after list all s3 files.
last_modified_end: datetime, optional
Filter the s3 files by the Last modified date of the object.
The filter is applied only after list all s3 files.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
Returns
-------
Dict[str, Dict[str, Any]]
Return a dictionary of objects returned from head_objects where the key is the object path.
The response object can be explored here:
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_object
Examples
--------
>>> import awswrangler as wr
>>> descs0 = wr.s3.describe_objects(['s3://bucket/key0', 's3://bucket/key1']) # Describe both objects
>>> descs1 = wr.s3.describe_objects('s3://bucket/prefix') # Describe all objects under the prefix
"""
paths: List[str] = _path2list(
path=path,
boto3_session=boto3_session,
last_modified_begin=last_modified_begin,
last_modified_end=last_modified_end,
)
if len(paths) < 1:
return {}
resp_list: List[Tuple[str, Dict[str, Any]]]
if len(paths) == 1:
resp_list = [
_describe_object(path=paths[0], boto3_session=boto3_session)
]
elif use_threads is False:
resp_list = [
_describe_object(path=p, boto3_session=boto3_session)
for p in paths
]
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
resp_list = list(
executor.map(
_describe_object_concurrent,
paths,
itertools.repeat(
_utils.boto3_to_primitives(
boto3_session=boto3_session)),
))
desc_dict: Dict[str, Dict[str, Any]] = dict(resp_list)
return desc_dict
def delete_objects(
path: Union[str, List[str]],
use_threads: bool = True,
last_modified_begin: Optional[datetime.datetime] = None,
last_modified_end: Optional[datetime.datetime] = None,
boto3_session: Optional[boto3.Session] = None,
) -> None:
"""Delete Amazon S3 objects from a received S3 prefix or list of S3 objects paths.
This function accepts Unix shell-style wildcards in the path argument.
* (matches everything), ? (matches any single character),
[seq] (matches any character in seq), [!seq] (matches any character not in seq).
Note
----
In case of `use_threads=True` the number of threads
that will be spawned will be gotten from os.cpu_count().
Note
----
The filter by last_modified begin last_modified end is applied after list all S3 files
Parameters
----------
path : Union[str, List[str]]
S3 prefix (accepts Unix shell-style wildcards)
(e.g. s3://bucket/prefix) or list of S3 objects paths (e.g. [s3://bucket/key0, s3://bucket/key1]).
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
last_modified_begin
Filter the s3 files by the Last modified date of the object.
The filter is applied only after list all s3 files.
last_modified_end: datetime, optional
Filter the s3 files by the Last modified date of the object.
The filter is applied only after list all s3 files.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
Returns
-------
None
None.
Examples
--------
>>> import awswrangler as wr
>>> wr.s3.delete_objects(['s3://bucket/key0', 's3://bucket/key1']) # Delete both objects
>>> wr.s3.delete_objects('s3://bucket/prefix') # Delete all objects under the received prefix
"""
paths: List[str] = _path2list(
path=path,
boto3_session=boto3_session,
last_modified_begin=last_modified_begin,
last_modified_end=last_modified_end,
)
if len(paths) < 1:
return
buckets: Dict[str, List[str]] = _split_paths_by_bucket(paths=paths)
for bucket, keys in buckets.items():
chunks: List[List[str]] = _utils.chunkify(lst=keys, max_length=1_000)
if len(chunks) == 1:
_delete_objects(bucket=bucket,
keys=chunks[0],
boto3_session=boto3_session)
elif use_threads is False:
for chunk in chunks:
_delete_objects(bucket=bucket,
keys=chunk,
boto3_session=boto3_session)
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
list(
executor.map(
_delete_objects_concurrent,
itertools.repeat(bucket),
chunks,
itertools.repeat(
_utils.boto3_to_primitives(
boto3_session=boto3_session)),
))
def _read_text(
parser_func: Callable,
path: Union[str, List[str]],
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, str]] = None,
chunksize: Optional[int] = None,
dataset: bool = False,
**pandas_kwargs,
) -> Union[pd.DataFrame, Iterator[pd.DataFrame]]:
if "iterator" in pandas_kwargs:
raise exceptions.InvalidArgument(
"Please, use chunksize instead of iterator.")
session: boto3.Session = _utils.ensure_session(session=boto3_session)
if (dataset is True) and (not isinstance(path, str)): # pragma: no cover
raise exceptions.InvalidArgument(
"The path argument must be a string Amazon S3 prefix if dataset=True."
)
if dataset is True:
path_root: str = str(path)
else:
path_root = ""
paths: List[str] = path2list(path=path, boto3_session=session)
_logger.debug("paths:\n%s", paths)
if chunksize is not None:
dfs: Iterator[pd.DataFrame] = _read_text_chunksize(
parser_func=parser_func,
paths=paths,
boto3_session=session,
chunksize=chunksize,
pandas_args=pandas_kwargs,
s3_additional_kwargs=s3_additional_kwargs,
dataset=dataset,
path_root=path_root,
)
return dfs
if use_threads is False:
df: pd.DataFrame = pd.concat(
objs=[
_read_text_full(
parser_func=parser_func,
path=p,
boto3_session=session,
pandas_args=pandas_kwargs,
s3_additional_kwargs=s3_additional_kwargs,
dataset=dataset,
path_root=path_root,
) for p in paths
],
ignore_index=True,
sort=False,
)
else:
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
with concurrent.futures.ThreadPoolExecutor(
max_workers=cpus) as executor:
df = pd.concat(
objs=executor.map(
_read_text_full,
itertools.repeat(parser_func),
itertools.repeat(path_root),
paths,
itertools.repeat(
_utils.boto3_to_primitives(
boto3_session=session)), # Boto3.Session
itertools.repeat(pandas_kwargs),
itertools.repeat(s3_additional_kwargs),
itertools.repeat(dataset),
),
ignore_index=True,
sort=False,
)
return df
def test_ensure_cpu_count(use_threads, result):
assert ensure_cpu_count(use_threads=use_threads) == result