def _validate_args(
df: pd.DataFrame,
table: Optional[str],
dataset: bool,
path: str,
partition_cols: Optional[List[str]],
mode: Optional[str],
description: Optional[str],
parameters: Optional[Dict[str, str]],
columns_comments: Optional[Dict[str, str]],
) -> None:
if df.empty is True:
raise exceptions.EmptyDataFrame()
if dataset is False:
if path.endswith("/"):
raise exceptions.InvalidArgumentValue(
"If <dataset=False>, the argument <path> should be a object path, not a directory."
)
if partition_cols:
raise exceptions.InvalidArgumentCombination(
"Please, pass dataset=True to be able to use partition_cols.")
if mode is not None:
raise exceptions.InvalidArgumentCombination(
"Please pass dataset=True to be able to use mode.")
if any(arg is not None
for arg in (table, description, parameters, columns_comments)):
raise exceptions.InvalidArgumentCombination(
"Please pass dataset=True to be able to use any one of these "
"arguments: database, table, description, parameters, "
"columns_comments.")
def _validate_args(
df: pd.DataFrame,
table: Optional[str],
database: Optional[str],
dataset: bool,
path: str,
partition_cols: Optional[List[str]],
mode: Optional[str],
description: Optional[str],
parameters: Optional[Dict[str, str]],
columns_comments: Optional[Dict[str, str]],
) -> None:
if df.empty is True:
raise exceptions.EmptyDataFrame()
if dataset is False:
if path.endswith("/"):
raise exceptions.InvalidArgumentValue(
"If <dataset=False>, the argument <path> should be a file path, not a directory."
)
if partition_cols:
raise exceptions.InvalidArgumentCombination(
"Please, pass dataset=True to be able to use partition_cols.")
if mode is not None:
raise exceptions.InvalidArgumentCombination(
"Please pass dataset=True to be able to use mode.")
if any(arg is not None
for arg in (table, description, parameters, columns_comments)):
raise exceptions.InvalidArgumentCombination(
"Please pass dataset=True to be able to use any one of these "
"arguments: database, table, description, parameters, "
"columns_comments.")
elif (database is None) != (table is None):
raise exceptions.InvalidArgumentCombination(
"Arguments database and table must be passed together. If you want to store your dataset metadata in "
"the Glue Catalog, please ensure you are passing both.")
def get_connection_attributes(
connection: Optional[str] = None,
secret_id: Optional[str] = None,
catalog_id: Optional[str] = None,
dbname: Optional[str] = None,
boto3_session: Optional[boto3.Session] = None,
connection_details: Optional[Dict[str,
str]] = None) -> ConnectionAttributes:
"""Get Connection Attributes."""
if connection is None and secret_id is None and connection_details is None:
raise exceptions.InvalidArgumentCombination(
"Failed attempt to connect. You MUST pass a connection name (Glue Catalog), a secret_id OR the connection_details as argument."
)
if connection is not None:
return _get_connection_attributes_from_catalog(
connection=connection,
catalog_id=catalog_id,
dbname=dbname,
boto3_session=boto3_session)
elif secret_id is not None:
return _get_connection_attributes_from_secrets_manager(
secret_id=cast(str, secret_id),
dbname=dbname,
boto3_session=boto3_session)
else:
return _get_connection_attributes_from_map(
connection_details=connection_details)
def _validate_args(
start_timestamp: int,
end_timestamp: int,
) -> None:
if start_timestamp < 0:
raise exceptions.InvalidArgument(
"`start_time` cannot be a negative value.")
if start_timestamp >= end_timestamp:
raise exceptions.InvalidArgumentCombination(
"`start_time` must be inferior to `end_time`.")
def _transaction_id(transaction_id: Optional[str] = None,
query_as_of_time: Optional[str] = None,
**kwargs: Any) -> Dict[str, Any]:
if transaction_id is not None and query_as_of_time is not None:
raise exceptions.InvalidArgumentCombination(
"Please pass only one of `transaction_id` or `query_as_of_time`, not both"
)
if transaction_id is not None:
kwargs["TransactionId"] = transaction_id
elif query_as_of_time is not None:
kwargs["QueryAsOfTime"] = query_as_of_time
return kwargs
def _get_default_logging_path(
subnet_id: Optional[str] = None,
account_id: Optional[str] = None,
region: Optional[str] = None,
boto3_session: Optional[boto3.Session] = None,
) -> str:
"""Get EMR default logging path.
E.g. "s3://aws-logs-{account_id}-{region}/elasticmapreduce/"
Parameters
----------
subnet_id : str, optional
Subnet ID. If not provided, you must pass `account_id` and `region` explicit.
account_id: str, optional
Account ID.
region: str, optional
Region e.g. 'us-east-1'
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
Returns
-------
str
Default logging path.
E.g. "s3://aws-logs-{account_id}-{region}/elasticmapreduce/"
Examples
--------
>>> import awswrangler as wr
>>> state = wr.emr._get_default_logging_path("subnet-id")
's3://aws-logs-{account_id}-{region}/elasticmapreduce/'
"""
if account_id is None:
boto3_session = _utils.ensure_session(session=boto3_session)
_account_id: str = _utils.get_account_id(boto3_session=boto3_session)
else:
_account_id = account_id
if (region is None) and (subnet_id is not None):
boto3_session = _utils.ensure_session(session=boto3_session)
_region: str = _utils.get_region_from_session(
boto3_session=boto3_session)
elif (region is None) and (subnet_id is None):
raise exceptions.InvalidArgumentCombination(
"You must pass region or subnet_id or both.")
else:
_region = region # type: ignore
return f"s3://aws-logs-{_account_id}-{_region}/elasticmapreduce/"
def _validate_args(
df: pd.DataFrame,
table: Optional[str],
database: Optional[str],
dataset: bool,
path: Optional[str],
partition_cols: Optional[List[str]],
bucketing_info: Optional[Tuple[List[str], int]],
mode: Optional[str],
description: Optional[str],
parameters: Optional[Dict[str, str]],
columns_comments: Optional[Dict[str, str]],
) -> None:
if df.empty is True:
raise exceptions.EmptyDataFrame("DataFrame cannot be empty.")
if dataset is False:
if path is None:
raise exceptions.InvalidArgumentValue(
"If dataset is False, the `path` argument must be passed.")
if path.endswith("/"):
raise exceptions.InvalidArgumentValue(
"If <dataset=False>, the argument <path> should be a key, not a prefix."
)
if partition_cols:
raise exceptions.InvalidArgumentCombination(
"Please, pass dataset=True to be able to use partition_cols.")
if bucketing_info:
raise exceptions.InvalidArgumentCombination(
"Please, pass dataset=True to be able to use bucketing_info.")
if mode is not None:
raise exceptions.InvalidArgumentCombination(
"Please pass dataset=True to be able to use mode.")
if any(arg is not None
for arg in (table, description, parameters, columns_comments)):
raise exceptions.InvalidArgumentCombination(
"Please pass dataset=True to be able to use any one of these "
"arguments: database, table, description, parameters, "
"columns_comments.")
elif (database is None) != (table is None):
raise exceptions.InvalidArgumentCombination(
"Arguments database and table must be passed together. If you want to store your dataset metadata in "
"the Glue Catalog, please ensure you are passing both.")
elif all(x is None for x in [path, database, table]):
raise exceptions.InvalidArgumentCombination(
"You must specify a `path` if dataset is True and database/table are not enabled."
)
elif bucketing_info and bucketing_info[1] <= 0:
raise exceptions.InvalidArgumentValue(
"Please pass a value greater than 1 for the number of buckets for bucketing."
)
def to_parquet( # pylint: disable=too-many-arguments,too-many-locals
df: pd.DataFrame,
path: str,
index: bool = False,
compression: Optional[str] = "snappy",
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, str]] = None,
sanitize_columns: bool = False,
dataset: bool = False,
partition_cols: Optional[List[str]] = None,
mode: Optional[str] = None,
catalog_versioning: bool = False,
database: Optional[str] = None,
table: Optional[str] = None,
dtype: Optional[Dict[str, str]] = None,
description: Optional[str] = None,
parameters: Optional[Dict[str, str]] = None,
columns_comments: Optional[Dict[str, str]] = None,
regular_partitions: bool = True,
projection_enabled: bool = False,
projection_types: Optional[Dict[str, str]] = None,
projection_ranges: Optional[Dict[str, str]] = None,
projection_values: Optional[Dict[str, str]] = None,
projection_intervals: Optional[Dict[str, str]] = None,
projection_digits: Optional[Dict[str, str]] = None,
) -> Dict[str, Union[List[str], Dict[str, List[str]]]]:
"""Write Parquet file or dataset on Amazon S3.
The concept of Dataset goes beyond the simple idea of files and enable more
complex features like partitioning, casting and catalog integration (Amazon Athena/AWS Glue Catalog).
Note
----
If `dataset=True` The table name and all column names will be automatically sanitized using
`wr.catalog.sanitize_table_name` and `wr.catalog.sanitize_column_name`.
Please, pass `sanitize_columns=True` to force the same behaviour for `dataset=False`.
Note
----
On `append` mode, the `parameters` will be upsert on an existing table.
Note
----
In case of `use_threads=True` the number of threads that will be spawned will be get from os.cpu_count().
Parameters
----------
df: pandas.DataFrame
Pandas DataFrame https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
path : str
S3 path (for file e.g. ``s3://bucket/prefix/filename.parquet``) (for dataset e.g. ``s3://bucket/prefix``).
index : bool
True to store the DataFrame index in file, otherwise False to ignore it.
compression: str, optional
Compression style (``None``, ``snappy``, ``gzip``).
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
s3_additional_kwargs:
Forward to s3fs, useful for server side encryption
https://s3fs.readthedocs.io/en/latest/#serverside-encryption
sanitize_columns : bool
True to sanitize columns names or False to keep it as is.
True value is forced if `dataset=True`.
dataset : bool
If True store a parquet dataset instead of a single file.
If True, enable all follow arguments:
partition_cols, mode, database, table, description, parameters, columns_comments, .
partition_cols: List[str], optional
List of column names that will be used to create partitions. Only takes effect if dataset=True.
mode: str, optional
``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True.
catalog_versioning : bool
If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it.
database : str, optional
Glue/Athena catalog: Database name.
table : str, optional
Glue/Athena catalog: Table name.
dtype : Dict[str, str], optional
Dictionary of columns names and Athena/Glue types to be casted.
Useful when you have columns with undetermined or mixed data types.
(e.g. {'col name': 'bigint', 'col2 name': 'int'})
description : str, optional
Glue/Athena catalog: Table description
parameters : Dict[str, str], optional
Glue/Athena catalog: Key/value pairs to tag the table.
columns_comments : Dict[str, str], optional
Glue/Athena catalog:
Columns names and the related comments (e.g. {'col0': 'Column 0.', 'col1': 'Column 1.', 'col2': 'Partition.'}).
regular_partitions : bool
Create regular partitions (Non projected partitions) on Glue Catalog.
Disable when you will work only with Partition Projection.
Keep enabled even when working with projections is useful to keep
Redshift Spectrum working with the regular partitions.
projection_enabled : bool
Enable Partition Projection on Athena (https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html)
projection_types : Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections types.
Valid types: "enum", "integer", "date", "injected"
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'enum', 'col2_name': 'integer'})
projection_ranges: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections ranges.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '0,10', 'col2_name': '-1,8675309'})
projection_values: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections values.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'A,B,Unknown', 'col2_name': 'foo,boo,bar'})
projection_intervals: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections intervals.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '5'})
projection_digits: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections digits.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '2'})
Returns
-------
Dict[str, Union[List[str], Dict[str, List[str]]]]
Dictionary with:
'paths': List of all stored files paths on S3.
'partitions_values': Dictionary of partitions added with keys as S3 path locations
and values as a list of partitions values as str.
Examples
--------
Writing single file
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.parquet',
... )
{
'paths': ['s3://bucket/prefix/my_file.parquet'],
'partitions_values': {}
}
Writing single file encrypted with a KMS key
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.parquet',
... s3_additional_kwargs={
... 'ServerSideEncryption': 'aws:kms',
... 'SSEKMSKeyId': 'YOUR_KMY_KEY_ARN'
... }
... )
{
'paths': ['s3://bucket/prefix/my_file.parquet'],
'partitions_values': {}
}
Writing partitioned dataset
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2']
... )
{
'paths': ['s3://.../col2=A/x.parquet', 's3://.../col2=B/y.parquet'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset to S3 with metadata on Athena/Glue Catalog.
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2'],
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... )
{
'paths': ['s3://.../col2=A/x.parquet', 's3://.../col2=B/y.parquet'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset casting empty column data type
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_parquet(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B'],
... 'col3': [None, None, None]
... }),
... path='s3://bucket/prefix',
... dataset=True,
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... dtype={'col3': 'date'}
... )
{
'paths': ['s3://.../x.parquet'],
'partitions_values: {}
}
"""
if (database is None) ^ (table is None):
raise exceptions.InvalidArgumentCombination(
"Please pass database and table arguments to be able to store the metadata into the Athena/Glue Catalog."
)
if df.empty is True:
raise exceptions.EmptyDataFrame()
partition_cols = partition_cols if partition_cols else []
dtype = dtype if dtype else {}
partitions_values: Dict[str, List[str]] = {}
# Sanitize table to respect Athena's standards
if (sanitize_columns is True) or (dataset is True):
df = catalog.sanitize_dataframe_columns_names(df=df)
partition_cols = [catalog.sanitize_column_name(p) for p in partition_cols]
dtype = {catalog.sanitize_column_name(k): v.lower() for k, v in dtype.items()}
catalog.drop_duplicated_columns(df=df)
session: boto3.Session = _utils.ensure_session(session=boto3_session)
cpus: int = _utils.ensure_cpu_count(use_threads=use_threads)
fs: s3fs.S3FileSystem = _utils.get_fs(session=session, s3_additional_kwargs=s3_additional_kwargs)
compression_ext: Optional[str] = _COMPRESSION_2_EXT.get(compression, None)
if compression_ext is None:
raise exceptions.InvalidCompression(f"{compression} is invalid, please use None, snappy or gzip.")
if dataset is False:
if path.endswith("/"): # pragma: no cover
raise exceptions.InvalidArgumentValue(
"If <dataset=False>, the argument <path> should be a object path, not a directory."
)
if partition_cols:
raise exceptions.InvalidArgumentCombination("Please, pass dataset=True to be able to use partition_cols.")
if mode is not None:
raise exceptions.InvalidArgumentCombination("Please pass dataset=True to be able to use mode.")
if any(arg is not None for arg in (database, table, description, parameters)):
raise exceptions.InvalidArgumentCombination(
"Please pass dataset=True to be able to use any one of these "
"arguments: database, table, description, parameters, "
"columns_comments."
)
df = _data_types.cast_pandas_with_athena_types(df=df, dtype=dtype)
schema: pa.Schema = _data_types.pyarrow_schema_from_pandas(
df=df, index=index, ignore_cols=partition_cols, dtype=dtype
)
_logger.debug("schema: \n%s", schema)
paths = [
_to_parquet_file(
df=df, path=path, schema=schema, index=index, compression=compression, cpus=cpus, fs=fs, dtype=dtype
)
]
else:
mode = "append" if mode is None else mode
if (
(mode in ("append", "overwrite_partitions")) and (database is not None) and (table is not None)
): # Fetching Catalog Types
catalog_types: Optional[Dict[str, str]] = catalog.get_table_types(
database=database, table=table, boto3_session=session
)
if catalog_types is not None:
for k, v in catalog_types.items():
dtype[k] = v
paths, partitions_values = _to_parquet_dataset(
df=df,
path=path,
index=index,
compression=compression,
compression_ext=compression_ext,
cpus=cpus,
fs=fs,
use_threads=use_threads,
partition_cols=partition_cols,
dtype=dtype,
mode=mode,
boto3_session=session,
)
if (database is not None) and (table is not None):
columns_types, partitions_types = _data_types.athena_types_from_pandas_partitioned(
df=df, index=index, partition_cols=partition_cols, dtype=dtype
)
catalog.create_parquet_table(
database=database,
table=table,
path=path,
columns_types=columns_types,
partitions_types=partitions_types,
compression=compression,
description=description,
parameters=parameters,
columns_comments=columns_comments,
boto3_session=session,
mode=mode,
catalog_versioning=catalog_versioning,
projection_enabled=projection_enabled,
projection_types=projection_types,
projection_ranges=projection_ranges,
projection_values=projection_values,
projection_intervals=projection_intervals,
projection_digits=projection_digits,
)
if partitions_values and (regular_partitions is True):
_logger.debug("partitions_values:\n%s", partitions_values)
catalog.add_parquet_partitions(
database=database,
table=table,
partitions_values=partitions_values,
compression=compression,
boto3_session=session,
)
return {"paths": paths, "partitions_values": partitions_values}
def to_csv( # pylint: disable=too-many-arguments,too-many-locals
df: pd.DataFrame,
path: str,
sep: str = ",",
index: bool = True,
columns: Optional[List[str]] = None,
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, str]] = None,
sanitize_columns: bool = False,
dataset: bool = False,
partition_cols: Optional[List[str]] = None,
mode: Optional[str] = None,
catalog_versioning: bool = False,
database: Optional[str] = None,
table: Optional[str] = None,
dtype: Optional[Dict[str, str]] = None,
description: Optional[str] = None,
parameters: Optional[Dict[str, str]] = None,
columns_comments: Optional[Dict[str, str]] = None,
regular_partitions: bool = True,
projection_enabled: bool = False,
projection_types: Optional[Dict[str, str]] = None,
projection_ranges: Optional[Dict[str, str]] = None,
projection_values: Optional[Dict[str, str]] = None,
projection_intervals: Optional[Dict[str, str]] = None,
projection_digits: Optional[Dict[str, str]] = None,
**pandas_kwargs,
) -> Dict[str, Union[List[str], Dict[str, List[str]]]]:
"""Write CSV file or dataset on Amazon S3.
The concept of Dataset goes beyond the simple idea of files and enable more
complex features like partitioning, casting and catalog integration (Amazon Athena/AWS Glue Catalog).
Note
----
If `dataset=True` The table name and all column names will be automatically sanitized using
`wr.catalog.sanitize_table_name` and `wr.catalog.sanitize_column_name`.
Please, pass `sanitize_columns=True` to force the same behaviour for `dataset=False`.
Note
----
If `dataset=True`, `pandas_kwargs` will be ignored due
restrictive quoting, date_format, escapechar, encoding, etc required by Athena/Glue Catalog.
Note
----
By now Pandas does not support in-memory CSV compression.
https://github.com/pandas-dev/pandas/issues/22555
So the `compression` will not be supported on Wrangler too.
Note
----
On `append` mode, the `parameters` will be upsert on an existing table.
Note
----
In case of `use_threads=True` the number of threads that will be spawned will be get from os.cpu_count().
Parameters
----------
df: pandas.DataFrame
Pandas DataFrame https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
path : str
Amazon S3 path (e.g. s3://bucket/filename.csv).
sep : str
String of length 1. Field delimiter for the output file.
index : bool
Write row names (index).
columns : List[str], optional
Columns to write.
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 Session will be used if boto3_session receive None.
s3_additional_kwargs:
Forward to s3fs, useful for server side encryption
https://s3fs.readthedocs.io/en/latest/#serverside-encryption
sanitize_columns : bool
True to sanitize columns names or False to keep it as is.
True value is forced if `dataset=True`.
dataset : bool
If True store a parquet dataset instead of a single file.
If True, enable all follow arguments:
partition_cols, mode, database, table, description, parameters, columns_comments, .
partition_cols: List[str], optional
List of column names that will be used to create partitions. Only takes effect if dataset=True.
mode : str, optional
``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True.
catalog_versioning : bool
If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it.
database : str, optional
Glue/Athena catalog: Database name.
table : str, optional
Glue/Athena catalog: Table name.
dtype : Dict[str, str], optional
Dictionary of columns names and Athena/Glue types to be casted.
Useful when you have columns with undetermined or mixed data types.
(e.g. {'col name': 'bigint', 'col2 name': 'int'})
description : str, optional
Glue/Athena catalog: Table description
parameters : Dict[str, str], optional
Glue/Athena catalog: Key/value pairs to tag the table.
columns_comments : Dict[str, str], optional
Glue/Athena catalog:
Columns names and the related comments (e.g. {'col0': 'Column 0.', 'col1': 'Column 1.', 'col2': 'Partition.'}).
regular_partitions : bool
Create regular partitions (Non projected partitions) on Glue Catalog.
Disable when you will work only with Partition Projection.
Keep enabled even when working with projections is useful to keep
Redshift Spectrum working with the regular partitions.
projection_enabled : bool
Enable Partition Projection on Athena (https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html)
projection_types : Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections types.
Valid types: "enum", "integer", "date", "injected"
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'enum', 'col2_name': 'integer'})
projection_ranges: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections ranges.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '0,10', 'col2_name': '-1,8675309'})
projection_values: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections values.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'A,B,Unknown', 'col2_name': 'foo,boo,bar'})
projection_intervals: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections intervals.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '5'})
projection_digits: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections digits.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '2'})
pandas_kwargs :
keyword arguments forwarded to pandas.DataFrame.to_csv()
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_csv.html
Returns
-------
None
None.
Examples
--------
Writing single file
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.csv',
... )
{
'paths': ['s3://bucket/prefix/my_file.csv'],
'partitions_values': {}
}
Writing single file encrypted with a KMS key
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.csv',
... s3_additional_kwargs={
... 'ServerSideEncryption': 'aws:kms',
... 'SSEKMSKeyId': 'YOUR_KMY_KEY_ARN'
... }
... )
{
'paths': ['s3://bucket/prefix/my_file.csv'],
'partitions_values': {}
}
Writing partitioned dataset
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2']
... )
{
'paths': ['s3://.../col2=A/x.csv', 's3://.../col2=B/y.csv'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset to S3 with metadata on Athena/Glue Catalog.
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2'],
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... )
{
'paths': ['s3://.../col2=A/x.csv', 's3://.../col2=B/y.csv'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset casting empty column data type
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B'],
... 'col3': [None, None, None]
... }),
... path='s3://bucket/prefix',
... dataset=True,
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... dtype={'col3': 'date'}
... )
{
'paths': ['s3://.../x.csv'],
'partitions_values: {}
}
"""
if (database is None) ^ (table is None):
raise exceptions.InvalidArgumentCombination(
"Please pass database and table arguments to be able to store the metadata into the Athena/Glue Catalog."
)
if df.empty is True:
raise exceptions.EmptyDataFrame()
partition_cols = partition_cols if partition_cols else []
dtype = dtype if dtype else {}
partitions_values: Dict[str, List[str]] = {}
# Sanitize table to respect Athena's standards
if (sanitize_columns is True) or (dataset is True):
df = catalog.sanitize_dataframe_columns_names(df=df)
partition_cols = [catalog.sanitize_column_name(p) for p in partition_cols]
dtype = {catalog.sanitize_column_name(k): v.lower() for k, v in dtype.items()}
catalog.drop_duplicated_columns(df=df)
session: boto3.Session = _utils.ensure_session(session=boto3_session)
fs: s3fs.S3FileSystem = _utils.get_fs(session=session, s3_additional_kwargs=s3_additional_kwargs)
if dataset is False:
if partition_cols:
raise exceptions.InvalidArgumentCombination("Please, pass dataset=True to be able to use partition_cols.")
if mode is not None:
raise exceptions.InvalidArgumentCombination("Please pass dataset=True to be able to use mode.")
if columns_comments:
raise exceptions.InvalidArgumentCombination("Please pass dataset=True to be able to use columns_comments.")
if any(arg is not None for arg in (database, table, description, parameters)):
raise exceptions.InvalidArgumentCombination(
"Please pass dataset=True to be able to use any one of these "
"arguments: database, table, description, parameters, "
"columns_comments."
)
pandas_kwargs["sep"] = sep
pandas_kwargs["index"] = index
pandas_kwargs["columns"] = columns
_to_text(file_format="csv", df=df, path=path, fs=fs, **pandas_kwargs)
paths = [path]
else:
mode = "append" if mode is None else mode
if columns:
df = df[columns]
if (
(mode in ("append", "overwrite_partitions")) and (database is not None) and (table is not None)
): # Fetching Catalog Types
catalog_types: Optional[Dict[str, str]] = catalog.get_table_types(
database=database, table=table, boto3_session=session
)
if catalog_types is not None:
for k, v in catalog_types.items():
dtype[k] = v
paths, partitions_values = _to_csv_dataset(
df=df,
path=path,
index=index,
sep=sep,
fs=fs,
use_threads=use_threads,
partition_cols=partition_cols,
dtype=dtype,
mode=mode,
boto3_session=session,
)
if (database is not None) and (table is not None):
columns_types, partitions_types = _data_types.athena_types_from_pandas_partitioned(
df=df, index=index, partition_cols=partition_cols, dtype=dtype, index_left=True
)
catalog.create_csv_table(
database=database,
table=table,
path=path,
columns_types=columns_types,
partitions_types=partitions_types,
description=description,
parameters=parameters,
columns_comments=columns_comments,
boto3_session=session,
mode=mode,
catalog_versioning=catalog_versioning,
sep=sep,
projection_enabled=projection_enabled,
projection_types=projection_types,
projection_ranges=projection_ranges,
projection_values=projection_values,
projection_intervals=projection_intervals,
projection_digits=projection_digits,
)
if partitions_values and (regular_partitions is True):
_logger.debug("partitions_values:\n%s", partitions_values)
catalog.add_csv_partitions(
database=database, table=table, partitions_values=partitions_values, boto3_session=session, sep=sep
)
return {"paths": paths, "partitions_values": partitions_values}
def to_csv( # pylint: disable=too-many-arguments,too-many-locals,too-many-statements
df: pd.DataFrame,
path: str,
sep: str = ",",
index: bool = True,
columns: Optional[List[str]] = None,
use_threads: bool = True,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, Any]] = None,
sanitize_columns: bool = False,
dataset: bool = False,
partition_cols: Optional[List[str]] = None,
bucketing_info: Optional[Tuple[List[str], int]] = None,
concurrent_partitioning: bool = False,
mode: Optional[str] = None,
catalog_versioning: bool = False,
database: Optional[str] = None,
table: Optional[str] = None,
dtype: Optional[Dict[str, str]] = None,
description: Optional[str] = None,
parameters: Optional[Dict[str, str]] = None,
columns_comments: Optional[Dict[str, str]] = None,
regular_partitions: bool = True,
projection_enabled: bool = False,
projection_types: Optional[Dict[str, str]] = None,
projection_ranges: Optional[Dict[str, str]] = None,
projection_values: Optional[Dict[str, str]] = None,
projection_intervals: Optional[Dict[str, str]] = None,
projection_digits: Optional[Dict[str, str]] = None,
catalog_id: Optional[str] = None,
**pandas_kwargs: Any,
) -> Dict[str, Union[List[str], Dict[str, List[str]]]]:
"""Write CSV file or dataset on Amazon S3.
The concept of Dataset goes beyond the simple idea of ordinary files and enable more
complex features like partitioning and catalog integration (Amazon Athena/AWS Glue Catalog).
Note
----
If database` and `table` arguments are passed, the table name and all column names
will be automatically sanitized using `wr.catalog.sanitize_table_name` and `wr.catalog.sanitize_column_name`.
Please, pass `sanitize_columns=True` to enforce this behaviour always.
Note
----
If `table` and `database` arguments are passed, `pandas_kwargs` will be ignored due
restrictive quoting, date_format, escapechar and encoding required by Athena/Glue Catalog.
Note
----
Compression: The minimum acceptable version to achive it is Pandas 1.2.0 that requires Python >= 3.7.1.
Note
----
On `append` mode, the `parameters` will be upsert on an existing table.
Note
----
In case of `use_threads=True` the number of threads
that will be spawned will be gotten from os.cpu_count().
Parameters
----------
df: pandas.DataFrame
Pandas DataFrame https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
path : str
Amazon S3 path (e.g. s3://bucket/filename.csv).
sep : str
String of length 1. Field delimiter for the output file.
index : bool
Write row names (index).
columns : Optional[List[str]]
Columns to write.
use_threads : bool
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 Session will be used if boto3_session receive None.
s3_additional_kwargs : Optional[Dict[str, Any]]
Forward to botocore requests. Valid parameters: "ACL", "Metadata", "ServerSideEncryption", "StorageClass",
"SSECustomerAlgorithm", "SSECustomerKey", "SSEKMSKeyId", "SSEKMSEncryptionContext", "Tagging", "RequestPayer".
e.g. s3_additional_kwargs={'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'}
sanitize_columns : bool
True to sanitize columns names or False to keep it as is.
True value is forced if `dataset=True`.
dataset : bool
If True store a parquet dataset instead of a ordinary file(s)
If True, enable all follow arguments:
partition_cols, mode, database, table, description, parameters, columns_comments, concurrent_partitioning,
catalog_versioning, projection_enabled, projection_types, projection_ranges, projection_values,
projection_intervals, projection_digits, catalog_id, schema_evolution.
partition_cols: List[str], optional
List of column names that will be used to create partitions. Only takes effect if dataset=True.
bucketing_info: Tuple[List[str], int], optional
Tuple consisting of the column names used for bucketing as the first element and the number of buckets as the
second element.
Only `str`, `int` and `bool` are supported as column data types for bucketing.
concurrent_partitioning: bool
If True will increase the parallelism level during the partitions writing. It will decrease the
writing time and increase the memory usage.
https://github.com/awslabs/aws-data-wrangler/blob/main/tutorials/022%20-%20Writing%20Partitions%20Concurrently.ipynb
mode : str, optional
``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True.
For details check the related tutorial:
https://aws-data-wrangler.readthedocs.io/en/2.4.0-docs/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet
catalog_versioning : bool
If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it.
database : str, optional
Glue/Athena catalog: Database name.
table : str, optional
Glue/Athena catalog: Table name.
dtype : Dict[str, str], optional
Dictionary of columns names and Athena/Glue types to be casted.
Useful when you have columns with undetermined or mixed data types.
(e.g. {'col name': 'bigint', 'col2 name': 'int'})
description : str, optional
Glue/Athena catalog: Table description
parameters : Dict[str, str], optional
Glue/Athena catalog: Key/value pairs to tag the table.
columns_comments : Dict[str, str], optional
Glue/Athena catalog:
Columns names and the related comments (e.g. {'col0': 'Column 0.', 'col1': 'Column 1.', 'col2': 'Partition.'}).
regular_partitions : bool
Create regular partitions (Non projected partitions) on Glue Catalog.
Disable when you will work only with Partition Projection.
Keep enabled even when working with projections is useful to keep
Redshift Spectrum working with the regular partitions.
projection_enabled : bool
Enable Partition Projection on Athena (https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html)
projection_types : Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections types.
Valid types: "enum", "integer", "date", "injected"
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'enum', 'col2_name': 'integer'})
projection_ranges: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections ranges.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '0,10', 'col2_name': '-1,8675309'})
projection_values: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections values.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'A,B,Unknown', 'col2_name': 'foo,boo,bar'})
projection_intervals: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections intervals.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '5'})
projection_digits: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections digits.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '2'})
catalog_id : str, optional
The ID of the Data Catalog from which to retrieve Databases.
If none is provided, the AWS account ID is used by default.
pandas_kwargs :
KEYWORD arguments forwarded to pandas.DataFrame.to_csv(). You can NOT pass `pandas_kwargs` explicit, just add
valid Pandas arguments in the function call and Wrangler will accept it.
e.g. wr.s3.to_csv(df, path, sep='|', na_rep='NULL', decimal=',')
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_csv.html
Returns
-------
Dict[str, Union[List[str], Dict[str, List[str]]]]
Dictionary with:
'paths': List of all stored files paths on S3.
'partitions_values': Dictionary of partitions added with keys as S3 path locations
and values as a list of partitions values as str.
Examples
--------
Writing single file
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.csv',
... )
{
'paths': ['s3://bucket/prefix/my_file.csv'],
'partitions_values': {}
}
Writing single file with pandas_kwargs
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.csv',
... sep='|',
... na_rep='NULL',
... decimal=','
... )
{
'paths': ['s3://bucket/prefix/my_file.csv'],
'partitions_values': {}
}
Writing single file encrypted with a KMS key
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/prefix/my_file.csv',
... s3_additional_kwargs={
... 'ServerSideEncryption': 'aws:kms',
... 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'
... }
... )
{
'paths': ['s3://bucket/prefix/my_file.csv'],
'partitions_values': {}
}
Writing partitioned dataset
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2']
... )
{
'paths': ['s3://.../col2=A/x.csv', 's3://.../col2=B/y.csv'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing bucketed dataset
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... bucketing_info=(["col2"], 2)
... )
{
'paths': ['s3://.../x_bucket-00000.csv', 's3://.../col2=B/x_bucket-00001.csv'],
'partitions_values: {}
}
Writing dataset to S3 with metadata on Athena/Glue Catalog.
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B']
... }),
... path='s3://bucket/prefix',
... dataset=True,
... partition_cols=['col2'],
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... )
{
'paths': ['s3://.../col2=A/x.csv', 's3://.../col2=B/y.csv'],
'partitions_values: {
's3://.../col2=A/': ['A'],
's3://.../col2=B/': ['B']
}
}
Writing dataset casting empty column data type
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_csv(
... df=pd.DataFrame({
... 'col': [1, 2, 3],
... 'col2': ['A', 'A', 'B'],
... 'col3': [None, None, None]
... }),
... path='s3://bucket/prefix',
... dataset=True,
... database='default', # Athena/Glue database
... table='my_table' # Athena/Glue table
... dtype={'col3': 'date'}
... )
{
'paths': ['s3://.../x.csv'],
'partitions_values: {}
}
"""
if "pandas_kwargs" in pandas_kwargs:
raise exceptions.InvalidArgument(
"You can NOT pass `pandas_kwargs` explicit, just add valid "
"Pandas arguments in the function call and Wrangler will accept it."
"e.g. wr.s3.to_csv(df, path, sep='|', na_rep='NULL', decimal=',', compression='gzip')"
)
if pandas_kwargs.get("compression") and str(pd.__version__) < LooseVersion("1.2.0"):
raise exceptions.InvalidArgument(
f"CSV compression on S3 is not supported for Pandas version {pd.__version__}. "
"The minimum acceptable version to achive it is Pandas 1.2.0 that requires Python >=3.7.1."
)
_validate_args(
df=df,
table=table,
database=database,
dataset=dataset,
path=path,
partition_cols=partition_cols,
bucketing_info=bucketing_info,
mode=mode,
description=description,
parameters=parameters,
columns_comments=columns_comments,
)
# Initializing defaults
partition_cols = partition_cols if partition_cols else []
dtype = dtype if dtype else {}
partitions_values: Dict[str, List[str]] = {}
mode = "append" if mode is None else mode
session: boto3.Session = _utils.ensure_session(session=boto3_session)
# Sanitize table to respect Athena's standards
if (sanitize_columns is True) or (database is not None and table is not None):
df, dtype, partition_cols = _sanitize(df=df, dtype=dtype, partition_cols=partition_cols)
# Evaluating dtype
catalog_table_input: Optional[Dict[str, Any]] = None
if database and table:
catalog_table_input = catalog._get_table_input( # pylint: disable=protected-access
database=database, table=table, boto3_session=session, catalog_id=catalog_id
)
if pandas_kwargs.get("compression") not in ("gzip", "bz2", None):
raise exceptions.InvalidArgumentCombination(
"If database and table are given, you must use one of these compressions: gzip, bz2 or None."
)
df = _apply_dtype(df=df, dtype=dtype, catalog_table_input=catalog_table_input, mode=mode)
if dataset is False:
pandas_kwargs["sep"] = sep
pandas_kwargs["index"] = index
pandas_kwargs["columns"] = columns
_to_text(
file_format="csv",
df=df,
use_threads=use_threads,
path=path,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
**pandas_kwargs,
)
paths = [path]
else:
if database and table:
quoting: Optional[int] = csv.QUOTE_NONE
escapechar: Optional[str] = "\\"
header: Union[bool, List[str]] = False
date_format: Optional[str] = "%Y-%m-%d %H:%M:%S.%f"
pd_kwargs: Dict[str, Any] = {}
compression: Optional[str] = pandas_kwargs.get("compression", None)
else:
quoting = pandas_kwargs.get("quoting", None)
escapechar = pandas_kwargs.get("escapechar", None)
header = pandas_kwargs.get("header", True)
date_format = pandas_kwargs.get("date_format", None)
compression = pandas_kwargs.get("compression", None)
pd_kwargs = pandas_kwargs.copy()
pd_kwargs.pop("quoting", None)
pd_kwargs.pop("escapechar", None)
pd_kwargs.pop("header", None)
pd_kwargs.pop("date_format", None)
pd_kwargs.pop("compression", None)
df = df[columns] if columns else df
paths, partitions_values = _to_dataset(
func=_to_text,
concurrent_partitioning=concurrent_partitioning,
df=df,
path_root=path,
index=index,
sep=sep,
compression=compression,
use_threads=use_threads,
partition_cols=partition_cols,
bucketing_info=bucketing_info,
mode=mode,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
file_format="csv",
quoting=quoting,
escapechar=escapechar,
header=header,
date_format=date_format,
**pd_kwargs,
)
if database and table:
try:
columns_types, partitions_types = _data_types.athena_types_from_pandas_partitioned(
df=df, index=index, partition_cols=partition_cols, dtype=dtype, index_left=True
)
catalog._create_csv_table( # pylint: disable=protected-access
database=database,
table=table,
path=path,
columns_types=columns_types,
partitions_types=partitions_types,
bucketing_info=bucketing_info,
description=description,
parameters=parameters,
columns_comments=columns_comments,
boto3_session=session,
mode=mode,
catalog_versioning=catalog_versioning,
sep=sep,
projection_enabled=projection_enabled,
projection_types=projection_types,
projection_ranges=projection_ranges,
projection_values=projection_values,
projection_intervals=projection_intervals,
projection_digits=projection_digits,
catalog_table_input=catalog_table_input,
catalog_id=catalog_id,
compression=pandas_kwargs.get("compression"),
skip_header_line_count=None,
)
if partitions_values and (regular_partitions is True):
_logger.debug("partitions_values:\n%s", partitions_values)
catalog.add_csv_partitions(
database=database,
table=table,
partitions_values=partitions_values,
bucketing_info=bucketing_info,
boto3_session=session,
sep=sep,
catalog_id=catalog_id,
columns_types=columns_types,
compression=pandas_kwargs.get("compression"),
)
except Exception:
_logger.debug("Catalog write failed, cleaning up S3 (paths: %s).", paths)
delete_objects(
path=paths,
use_threads=use_threads,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
)
raise
return {"paths": paths, "partitions_values": partitions_values}
def to_json( # pylint: disable=too-many-arguments,too-many-locals,too-many-statements,too-many-branches
df: pd.DataFrame,
path: Optional[str] = None,
index: bool = True,
columns: Optional[List[str]] = None,
use_threads: Union[bool, int] = True,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, Any]] = None,
sanitize_columns: bool = False,
dataset: bool = False,
filename_prefix: Optional[str] = None,
partition_cols: Optional[List[str]] = None,
bucketing_info: Optional[Tuple[List[str], int]] = None,
concurrent_partitioning: bool = False,
mode: Optional[str] = None,
catalog_versioning: bool = False,
schema_evolution: bool = True,
database: Optional[str] = None,
table: Optional[str] = None,
table_type: Optional[str] = None,
transaction_id: Optional[str] = None,
dtype: Optional[Dict[str, str]] = None,
description: Optional[str] = None,
parameters: Optional[Dict[str, str]] = None,
columns_comments: Optional[Dict[str, str]] = None,
regular_partitions: bool = True,
projection_enabled: bool = False,
projection_types: Optional[Dict[str, str]] = None,
projection_ranges: Optional[Dict[str, str]] = None,
projection_values: Optional[Dict[str, str]] = None,
projection_intervals: Optional[Dict[str, str]] = None,
projection_digits: Optional[Dict[str, str]] = None,
catalog_id: Optional[str] = None,
**pandas_kwargs: Any,
) -> Union[List[str], Dict[str, Union[List[str], Dict[str, List[str]]]]]:
"""Write JSON file on Amazon S3.
Note
----
In case of `use_threads=True` the number of threads
that will be spawned will be gotten from os.cpu_count().
Note
----
Compression: The minimum acceptable version to achive it is Pandas 1.2.0 that requires Python >= 3.7.1.
Parameters
----------
df: pandas.DataFrame
Pandas DataFrame https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html
path : str
Amazon S3 path (e.g. s3://bucket/filename.json).
index : bool
Write row names (index).
columns : Optional[List[str]]
Columns to write.
use_threads : bool, int
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() will be used as the max number of threads.
If integer is provided, specified number is used.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 Session will be used if boto3_session receive None.
s3_additional_kwargs : Optional[Dict[str, Any]]
Forwarded to botocore requests.
e.g. s3_additional_kwargs={'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'}
sanitize_columns : bool
True to sanitize columns names or False to keep it as is.
True value is forced if `dataset=True`.
dataset : bool
If True store as a dataset instead of ordinary file(s)
If True, enable all follow arguments:
partition_cols, mode, database, table, description, parameters, columns_comments, concurrent_partitioning,
catalog_versioning, projection_enabled, projection_types, projection_ranges, projection_values,
projection_intervals, projection_digits, catalog_id, schema_evolution.
filename_prefix: str, optional
If dataset=True, add a filename prefix to the output files.
partition_cols: List[str], optional
List of column names that will be used to create partitions. Only takes effect if dataset=True.
bucketing_info: Tuple[List[str], int], optional
Tuple consisting of the column names used for bucketing as the first element and the number of buckets as the
second element.
Only `str`, `int` and `bool` are supported as column data types for bucketing.
concurrent_partitioning: bool
If True will increase the parallelism level during the partitions writing. It will decrease the
writing time and increase the memory usage.
https://aws-data-wrangler.readthedocs.io/en/2.13.0/tutorials/022%20-%20Writing%20Partitions%20Concurrently.html
mode : str, optional
``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True.
For details check the related tutorial:
https://aws-data-wrangler.readthedocs.io/en/2.13.0/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet
catalog_versioning : bool
If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it.
schema_evolution : bool
If True allows schema evolution (new or missing columns), otherwise a exception will be raised.
(Only considered if dataset=True and mode in ("append", "overwrite_partitions"))
Related tutorial:
https://aws-data-wrangler.readthedocs.io/en/2.13.0/tutorials/014%20-%20Schema%20Evolution.html
database : str, optional
Glue/Athena catalog: Database name.
table : str, optional
Glue/Athena catalog: Table name.
table_type: str, optional
The type of the Glue Table. Set to EXTERNAL_TABLE if None
transaction_id: str, optional
The ID of the transaction when writing to a Governed Table.
dtype : Dict[str, str], optional
Dictionary of columns names and Athena/Glue types to be casted.
Useful when you have columns with undetermined or mixed data types.
(e.g. {'col name': 'bigint', 'col2 name': 'int'})
description : str, optional
Glue/Athena catalog: Table description
parameters : Dict[str, str], optional
Glue/Athena catalog: Key/value pairs to tag the table.
columns_comments : Dict[str, str], optional
Glue/Athena catalog:
Columns names and the related comments (e.g. {'col0': 'Column 0.', 'col1': 'Column 1.', 'col2': 'Partition.'}).
regular_partitions : bool
Create regular partitions (Non projected partitions) on Glue Catalog.
Disable when you will work only with Partition Projection.
Keep enabled even when working with projections is useful to keep
Redshift Spectrum working with the regular partitions.
projection_enabled : bool
Enable Partition Projection on Athena (https://docs.aws.amazon.com/athena/latest/ug/partition-projection.html)
projection_types : Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections types.
Valid types: "enum", "integer", "date", "injected"
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'enum', 'col2_name': 'integer'})
projection_ranges: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections ranges.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '0,10', 'col2_name': '-1,8675309'})
projection_values: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections values.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': 'A,B,Unknown', 'col2_name': 'foo,boo,bar'})
projection_intervals: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections intervals.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '5'})
projection_digits: Optional[Dict[str, str]]
Dictionary of partitions names and Athena projections digits.
https://docs.aws.amazon.com/athena/latest/ug/partition-projection-supported-types.html
(e.g. {'col_name': '1', 'col2_name': '2'})
catalog_id : str, optional
The ID of the Data Catalog from which to retrieve Databases.
If none is provided, the AWS account ID is used by default.
pandas_kwargs:
KEYWORD arguments forwarded to pandas.DataFrame.to_json(). You can NOT pass `pandas_kwargs` explicit, just add
valid Pandas arguments in the function call and Wrangler will accept it.
e.g. wr.s3.to_json(df, path, lines=True, date_format='iso')
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_json.html
Returns
-------
List[str]
List of written files.
Examples
--------
Writing JSON file
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_json(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/filename.json',
... )
Writing JSON file using pandas_kwargs
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_json(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/filename.json',
... lines=True,
... date_format='iso'
... )
Writing CSV file encrypted with a KMS key
>>> import awswrangler as wr
>>> import pandas as pd
>>> wr.s3.to_json(
... df=pd.DataFrame({'col': [1, 2, 3]}),
... path='s3://bucket/filename.json',
... s3_additional_kwargs={
... 'ServerSideEncryption': 'aws:kms',
... 'SSEKMSKeyId': 'YOUR_KMS_KEY_ARN'
... }
... )
"""
if "pandas_kwargs" in pandas_kwargs:
raise exceptions.InvalidArgument(
"You can NOT pass `pandas_kwargs` explicit, just add valid "
"Pandas arguments in the function call and Wrangler will accept it."
"e.g. wr.s3.to_json(df, path, lines=True, date_format='iso')")
if pandas_kwargs.get("compression") and str(
pd.__version__) < LooseVersion("1.2.0"):
raise exceptions.InvalidArgument(
f"JSON compression on S3 is not supported for Pandas version {pd.__version__}. "
"The minimum acceptable version to achive it is Pandas 1.2.0 that requires Python >=3.7.1."
)
_validate_args(
df=df,
table=table,
database=database,
dataset=dataset,
path=path,
partition_cols=partition_cols,
bucketing_info=bucketing_info,
mode=mode,
description=description,
parameters=parameters,
columns_comments=columns_comments,
)
# Initializing defaults
partition_cols = partition_cols if partition_cols else []
dtype = dtype if dtype else {}
partitions_values: Dict[str, List[str]] = {}
mode = "append" if mode is None else mode
commit_trans: bool = False
if transaction_id:
table_type = "GOVERNED"
filename_prefix = filename_prefix + uuid.uuid4(
).hex if filename_prefix else uuid.uuid4().hex
session: boto3.Session = _utils.ensure_session(session=boto3_session)
# Sanitize table to respect Athena's standards
if (sanitize_columns is True) or (database is not None
and table is not None):
df, dtype, partition_cols = _sanitize(df=df,
dtype=dtype,
partition_cols=partition_cols)
# Evaluating dtype
catalog_table_input: Optional[Dict[str, Any]] = None
if database and table:
catalog_table_input = catalog._get_table_input( # pylint: disable=protected-access
database=database,
table=table,
boto3_session=session,
transaction_id=transaction_id,
catalog_id=catalog_id)
catalog_path: Optional[str] = None
if catalog_table_input:
table_type = catalog_table_input["TableType"]
catalog_path = catalog_table_input.get("StorageDescriptor",
{}).get("Location")
if path is None:
if catalog_path:
path = catalog_path
else:
raise exceptions.InvalidArgumentValue(
"Glue table does not exist in the catalog. Please pass the `path` argument to create it."
)
elif path and catalog_path:
if path.rstrip("/") != catalog_path.rstrip("/"):
raise exceptions.InvalidArgumentValue(
f"The specified path: {path}, does not match the existing Glue catalog table path: {catalog_path}"
)
if pandas_kwargs.get("compression") not in ("gzip", "bz2", None):
raise exceptions.InvalidArgumentCombination(
"If database and table are given, you must use one of these compressions: gzip, bz2 or None."
)
if (table_type == "GOVERNED") and (not transaction_id):
_logger.debug(
"`transaction_id` not specified for GOVERNED table, starting transaction"
)
transaction_id = lakeformation.start_transaction(
read_only=False, boto3_session=boto3_session)
commit_trans = True
df = _apply_dtype(df=df,
dtype=dtype,
catalog_table_input=catalog_table_input,
mode=mode)
if dataset is False:
return _to_text(
file_format="json",
df=df,
path=path,
use_threads=use_threads,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
**pandas_kwargs,
)
compression: Optional[str] = pandas_kwargs.get("compression", None)
df = df[columns] if columns else df
columns_types: Dict[str, str] = {}
partitions_types: Dict[str, str] = {}
if database and table:
columns_types, partitions_types = _data_types.athena_types_from_pandas_partitioned(
df=df, index=index, partition_cols=partition_cols, dtype=dtype)
if schema_evolution is False:
_utils.check_schema_changes(columns_types=columns_types,
table_input=catalog_table_input,
mode=mode)
if (catalog_table_input is None) and (table_type == "GOVERNED"):
catalog._create_json_table( # pylint: disable=protected-access
database=database,
table=table,
path=path, # type: ignore
columns_types=columns_types,
table_type=table_type,
partitions_types=partitions_types,
bucketing_info=bucketing_info,
description=description,
parameters=parameters,
columns_comments=columns_comments,
boto3_session=session,
mode=mode,
transaction_id=transaction_id,
catalog_versioning=catalog_versioning,
schema_evolution=schema_evolution,
projection_enabled=projection_enabled,
projection_types=projection_types,
projection_ranges=projection_ranges,
projection_values=projection_values,
projection_intervals=projection_intervals,
projection_digits=projection_digits,
projection_storage_location_template=None,
catalog_table_input=catalog_table_input,
catalog_id=catalog_id,
compression=pandas_kwargs.get("compression"),
serde_library=None,
serde_parameters=None,
)
catalog_table_input = catalog._get_table_input( # pylint: disable=protected-access
database=database,
table=table,
boto3_session=session,
transaction_id=transaction_id,
catalog_id=catalog_id,
)
paths, partitions_values = _to_dataset(
func=_to_text,
concurrent_partitioning=concurrent_partitioning,
df=df,
path_root=path, # type: ignore
filename_prefix=filename_prefix,
index=index,
compression=compression,
catalog_id=catalog_id,
database=database,
table=table,
table_type=table_type,
transaction_id=transaction_id,
use_threads=use_threads,
partition_cols=partition_cols,
partitions_types=partitions_types,
bucketing_info=bucketing_info,
mode=mode,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
file_format="json",
)
if database and table:
try:
serde_info: Dict[str, Any] = {}
if catalog_table_input:
serde_info = catalog_table_input["StorageDescriptor"][
"SerdeInfo"]
serde_library: Optional[str] = serde_info.get(
"SerializationLibrary", None)
serde_parameters: Optional[Dict[str, str]] = serde_info.get(
"Parameters", None)
catalog._create_json_table( # pylint: disable=protected-access
database=database,
table=table,
path=path, # type: ignore
columns_types=columns_types,
table_type=table_type,
partitions_types=partitions_types,
bucketing_info=bucketing_info,
description=description,
parameters=parameters,
columns_comments=columns_comments,
boto3_session=session,
mode=mode,
transaction_id=transaction_id,
catalog_versioning=catalog_versioning,
schema_evolution=schema_evolution,
projection_enabled=projection_enabled,
projection_types=projection_types,
projection_ranges=projection_ranges,
projection_values=projection_values,
projection_intervals=projection_intervals,
projection_digits=projection_digits,
projection_storage_location_template=None,
catalog_table_input=catalog_table_input,
catalog_id=catalog_id,
compression=pandas_kwargs.get("compression"),
serde_library=serde_library,
serde_parameters=serde_parameters,
)
if partitions_values and (regular_partitions is
True) and (table_type != "GOVERNED"):
_logger.debug("partitions_values:\n%s", partitions_values)
catalog.add_json_partitions(
database=database,
table=table,
partitions_values=partitions_values,
bucketing_info=bucketing_info,
boto3_session=session,
serde_library=serde_library,
serde_parameters=serde_parameters,
catalog_id=catalog_id,
columns_types=columns_types,
compression=pandas_kwargs.get("compression"),
)
if commit_trans:
lakeformation.commit_transaction(
transaction_id=transaction_id,
boto3_session=boto3_session # type: ignore
)
except Exception:
_logger.debug("Catalog write failed, cleaning up S3 (paths: %s).",
paths)
delete_objects(
path=paths,
use_threads=use_threads,
boto3_session=session,
s3_additional_kwargs=s3_additional_kwargs,
)
raise
return {"paths": paths, "partitions_values": partitions_values}
def _create_table( # pylint: disable=too-many-branches,too-many-statements
database: str,
table: str,
description: Optional[str],
parameters: Optional[Dict[str, str]],
mode: str,
catalog_versioning: bool,
boto3_session: Optional[boto3.Session],
table_input: Dict[str, Any],
table_exist: bool,
projection_enabled: bool,
partitions_types: Optional[Dict[str, str]],
columns_comments: Optional[Dict[str, str]],
projection_types: Optional[Dict[str, str]],
projection_ranges: Optional[Dict[str, str]],
projection_values: Optional[Dict[str, str]],
projection_intervals: Optional[Dict[str, str]],
projection_digits: Optional[Dict[str, str]],
catalog_id: Optional[str],
) -> None:
# Description
mode = _update_if_necessary(dic=table_input,
key="Description",
value=description,
mode=mode)
# Parameters
parameters = parameters if parameters else {}
for k, v in parameters.items():
mode = _update_if_necessary(dic=table_input["Parameters"],
key=k,
value=v,
mode=mode)
# Projection
if projection_enabled is True:
table_input["Parameters"]["projection.enabled"] = "true"
partitions_types = partitions_types if partitions_types else {}
projection_types = projection_types if projection_types else {}
projection_ranges = projection_ranges if projection_ranges else {}
projection_values = projection_values if projection_values else {}
projection_intervals = projection_intervals if projection_intervals else {}
projection_digits = projection_digits if projection_digits else {}
projection_types = {
sanitize_column_name(k): v
for k, v in projection_types.items()
}
projection_ranges = {
sanitize_column_name(k): v
for k, v in projection_ranges.items()
}
projection_values = {
sanitize_column_name(k): v
for k, v in projection_values.items()
}
projection_intervals = {
sanitize_column_name(k): v
for k, v in projection_intervals.items()
}
projection_digits = {
sanitize_column_name(k): v
for k, v in projection_digits.items()
}
for k, v in projection_types.items():
dtype: Optional[str] = partitions_types.get(k)
if dtype is None:
raise exceptions.InvalidArgumentCombination(
f"Column {k} appears as projected column but not as partitioned column."
)
if dtype == "date":
table_input["Parameters"][
f"projection.{k}.format"] = "yyyy-MM-dd"
elif dtype == "timestamp":
table_input["Parameters"][
f"projection.{k}.format"] = "yyyy-MM-dd HH:mm:ss"
table_input["Parameters"][
f"projection.{k}.interval.unit"] = "SECONDS"
table_input["Parameters"][f"projection.{k}.interval"] = "1"
for k, v in projection_types.items():
mode = _update_if_necessary(dic=table_input["Parameters"],
key=f"projection.{k}.type",
value=v,
mode=mode)
for k, v in projection_ranges.items():
mode = _update_if_necessary(dic=table_input["Parameters"],
key=f"projection.{k}.range",
value=v,
mode=mode)
for k, v in projection_values.items():
mode = _update_if_necessary(dic=table_input["Parameters"],
key=f"projection.{k}.values",
value=v,
mode=mode)
for k, v in projection_intervals.items():
mode = _update_if_necessary(dic=table_input["Parameters"],
key=f"projection.{k}.interval",
value=str(v),
mode=mode)
for k, v in projection_digits.items():
mode = _update_if_necessary(dic=table_input["Parameters"],
key=f"projection.{k}.digits",
value=str(v),
mode=mode)
else:
table_input["Parameters"]["projection.enabled"] = "false"
# Column comments
columns_comments = columns_comments if columns_comments else {}
columns_comments = {
sanitize_column_name(k): v
for k, v in columns_comments.items()
}
if columns_comments:
for col in table_input["StorageDescriptor"]["Columns"]:
name: str = col["Name"]
if name in columns_comments:
mode = _update_if_necessary(dic=col,
key="Comment",
value=columns_comments[name],
mode=mode)
for par in table_input["PartitionKeys"]:
name = par["Name"]
if name in columns_comments:
mode = _update_if_necessary(dic=par,
key="Comment",
value=columns_comments[name],
mode=mode)
_logger.debug("table_input: %s", table_input)
session: boto3.Session = _utils.ensure_session(session=boto3_session)
client_glue: boto3.client = _utils.client(service_name="glue",
session=session)
skip_archive: bool = not catalog_versioning
if mode not in ("overwrite", "append", "overwrite_partitions", "update"):
raise exceptions.InvalidArgument(
f"{mode} is not a valid mode. It must be 'overwrite', 'append' or 'overwrite_partitions'."
)
if table_exist is True and mode == "overwrite":
delete_all_partitions(table=table,
database=database,
catalog_id=catalog_id,
boto3_session=session)
_logger.debug("Updating table (%s)...", mode)
client_glue.update_table(**_catalog_id(catalog_id=catalog_id,
DatabaseName=database,
TableInput=table_input,
SkipArchive=skip_archive))
elif (table_exist is True) and (mode in ("append", "overwrite_partitions",
"update")):
if mode == "update":
_logger.debug("Updating table (%s)...", mode)
client_glue.update_table(**_catalog_id(catalog_id=catalog_id,
DatabaseName=database,
TableInput=table_input,
SkipArchive=skip_archive))
elif table_exist is False:
try:
_logger.debug("Creating table (%s)...", mode)
client_glue.create_table(**_catalog_id(catalog_id=catalog_id,
DatabaseName=database,
TableInput=table_input))
except client_glue.exceptions.AlreadyExistsException:
if mode == "overwrite":
_utils.try_it(
f=_overwrite_table,
ex=client_glue.exceptions.AlreadyExistsException,
client_glue=client_glue,
catalog_id=catalog_id,
database=database,
table=table,
table_input=table_input,
boto3_session=boto3_session,
)
_logger.debug("Leaving table as is (%s)...", mode)
def select_query(
sql: str,
path: str,
input_serialization: str,
input_serialization_params: Dict[str, Union[bool, str]],
compression: Optional[str] = None,
use_threads: Union[bool, int] = False,
boto3_session: Optional[boto3.Session] = None,
s3_additional_kwargs: Optional[Dict[str, Any]] = None,
) -> pd.DataFrame:
r"""Filter contents of an Amazon S3 object based on SQL statement.
Note: Scan ranges are only supported for uncompressed CSV/JSON, CSV (without quoted delimiters)
and JSON objects (in LINES mode only). It means scanning cannot be split across threads if the latter
conditions are not met, leading to lower performance.
Parameters
----------
sql: str
SQL statement used to query the object.
path: str
S3 path to the object (e.g. s3://bucket/key).
input_serialization: str,
Format of the S3 object queried.
Valid values: "CSV", "JSON", or "Parquet". Case sensitive.
input_serialization_params: Dict[str, Union[bool, str]]
Dictionary describing the serialization of the S3 object.
compression: Optional[str]
Compression type of the S3 object.
Valid values: None, "gzip", or "bzip2". gzip and bzip2 are only valid for CSV and JSON objects.
use_threads : Union[bool, int]
True to enable concurrent requests, False to disable multiple threads.
If enabled os.cpu_count() is used as the max number of threads.
If integer is provided, specified number is used.
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session is used if none is provided.
s3_additional_kwargs : Optional[Dict[str, Any]]
Forwarded to botocore requests.
Valid values: "SSECustomerAlgorithm", "SSECustomerKey", "ExpectedBucketOwner".
e.g. s3_additional_kwargs={'SSECustomerAlgorithm': 'md5'}
Returns
-------
pandas.DataFrame
Pandas DataFrame with results from query.
Examples
--------
Reading a gzip compressed JSON document
>>> import awswrangler as wr
>>> df = wr.s3.select_query(
... sql='SELECT * FROM s3object[*][*]',
... path='s3://bucket/key.json.gzip',
... input_serialization='JSON',
... input_serialization_params={
... 'Type': 'Document',
... },
... compression="gzip",
... )
Reading an entire CSV object using threads
>>> import awswrangler as wr
>>> df = wr.s3.select_query(
... sql='SELECT * FROM s3object',
... path='s3://bucket/key.csv',
... input_serialization='CSV',
... input_serialization_params={
... 'FileHeaderInfo': 'Use',
... 'RecordDelimiter': '\r\n'
... },
... use_threads=True,
... )
Reading a single column from Parquet object with pushdown filter
>>> import awswrangler as wr
>>> df = wr.s3.select_query(
... sql='SELECT s.\"id\" FROM s3object s where s.\"id\" = 1.0',
... path='s3://bucket/key.snappy.parquet',
... input_serialization='Parquet',
... )
"""
if path.endswith("/"):
raise exceptions.InvalidArgumentValue("<path> argument should be an S3 key, not a prefix.")
if input_serialization not in ["CSV", "JSON", "Parquet"]:
raise exceptions.InvalidArgumentValue("<input_serialization> argument must be 'CSV', 'JSON' or 'Parquet'")
if compression not in [None, "gzip", "bzip2"]:
raise exceptions.InvalidCompression(f"Invalid {compression} compression, please use None, 'gzip' or 'bzip2'.")
if compression and (input_serialization not in ["CSV", "JSON"]):
raise exceptions.InvalidArgumentCombination(
"'gzip' or 'bzip2' are only valid for input 'CSV' or 'JSON' objects."
)
bucket, key = _utils.parse_path(path)
args: Dict[str, Any] = {
"Bucket": bucket,
"Key": key,
"Expression": sql,
"ExpressionType": "SQL",
"RequestProgress": {"Enabled": False},
"InputSerialization": {
input_serialization: input_serialization_params,
"CompressionType": compression.upper() if compression else "NONE",
},
"OutputSerialization": {
"JSON": {},
},
}
if s3_additional_kwargs:
args.update(s3_additional_kwargs)
_logger.debug("args:\n%s", pprint.pformat(args))
if any(
[
compression,
input_serialization_params.get("AllowQuotedRecordDelimiter"),
input_serialization_params.get("Type") == "Document",
]
): # Scan range is only supported for uncompressed CSV/JSON, CSV (without quoted delimiters)
# and JSON objects (in LINES mode only)
_logger.debug("Scan ranges are not supported given provided input.")
return pd.DataFrame(_select_object_content(args=args, boto3_session=boto3_session))
return _paginate_stream(args=args, path=path, use_threads=use_threads, boto3_session=boto3_session)
def get_tables(
catalog_id: Optional[str] = None,
database: Optional[str] = None,
name_contains: Optional[str] = None,
name_prefix: Optional[str] = None,
name_suffix: Optional[str] = None,
boto3_session: Optional[boto3.Session] = None,
) -> Iterator[Dict[str, Any]]:
"""Get an iterator of tables.
Note
----
Please, does not filter using name_contains and name_prefix/name_suffix at the same time.
Only name_prefix and name_suffix can be combined together.
Parameters
----------
catalog_id : str, optional
The ID of the Data Catalog from which to retrieve Databases.
If none is provided, the AWS account ID is used by default.
database : str, optional
Database name.
name_contains : str, optional
Select by a specific string on table name
name_prefix : str, optional
Select by a specific prefix on table name
name_suffix : str, optional
Select by a specific suffix on table name
boto3_session : boto3.Session(), optional
Boto3 Session. The default boto3 session will be used if boto3_session receive None.
Returns
-------
Iterator[Dict[str, Any]]
Iterator of tables.
Examples
--------
>>> import awswrangler as wr
>>> tables = wr.catalog.get_tables()
"""
client_glue: boto3.client = _utils.client(service_name="glue",
session=boto3_session)
paginator = client_glue.get_paginator("get_tables")
args: Dict[str, str] = {}
if catalog_id is not None:
args["CatalogId"] = catalog_id
if (name_prefix is not None) and (name_suffix
is not None) and (name_contains
is not None):
raise exceptions.InvalidArgumentCombination(
"Please, does not filter using name_contains and "
"name_prefix/name_suffix at the same time. Only "
"name_prefix and name_suffix can be combined together.")
if (name_prefix is not None) and (name_suffix is not None):
args["Expression"] = f"{name_prefix}*{name_suffix}"
elif name_contains is not None:
args["Expression"] = f"*{name_contains}*"
elif name_prefix is not None:
args["Expression"] = f"{name_prefix}*"
elif name_suffix is not None:
args["Expression"] = f"*{name_suffix}"
if database is not None:
dbs: List[str] = [database]
else:
dbs = [x["Name"] for x in get_databases(catalog_id=catalog_id)]
for db in dbs:
args["DatabaseName"] = db
response_iterator = paginator.paginate(**args)
try:
for page in response_iterator:
for tbl in page["TableList"]:
yield tbl
except client_glue.exceptions.EntityNotFoundException:
continue
