def to_pickle_distributed(cls, qc, **kwargs):
"""
When `*` in the filename all partitions are written to their own separate file.
The filenames is determined as follows:
- if `*` in the filename then it will be replaced by the increasing sequence 0, 1, 2, …
- if `*` is not the filename, then will be used default implementation.
Examples #1: 4 partitions and input filename="partition*.pkl.gz", then filenames will be:
`partition0.pkl.gz`, `partition1.pkl.gz`, `partition2.pkl.gz`, `partition3.pkl.gz`.
Parameters
----------
qc : BaseQueryCompiler
The query compiler of the Modin dataframe that we want
to run ``to_pickle_distributed`` on.
**kwargs : dict
Parameters for ``pandas.to_pickle(**kwargs)``.
"""
if not (isinstance(kwargs["filepath_or_buffer"], str)
and "*" in kwargs["filepath_or_buffer"]) or not isinstance(
qc, PandasQueryCompiler):
warnings.warn("Defaulting to Modin core implementation")
return PandasOnRayIO.to_pickle(qc, **kwargs)
def func(df, **kw):
idx = str(kw["partition_idx"])
kwargs["path"] = kwargs.pop("filepath_or_buffer").replace("*", idx)
df.to_pickle(**kwargs)
return pandas.DataFrame()
result = qc._modin_frame.broadcast_apply_full_axis(
1,
func,
other=None,
new_index=[],
new_columns=[],
enumerate_partitions=True)
result.to_pandas()
def read_sql(
cls,
sql,
con,
index_col=None,
coerce_float=True,
params=None,
parse_dates=None,
columns=None,
chunksize=None,
partition_column=None,
lower_bound=None,
upper_bound=None,
):
""" Read SQL query or database table into a DataFrame.
Args:
sql: string or SQLAlchemy Selectable (select or text object) SQL query to be executed or a table name.
con: SQLAlchemy connectable (engine/connection) or database string URI or DBAPI2 connection (fallback mode)
index_col: Column(s) to set as index(MultiIndex).
coerce_float: Attempts to convert values of non-string, non-numeric objects (like decimal.Decimal) to
floating point, useful for SQL result sets.
params: List of parameters to pass to execute method. The syntax used
to pass parameters is database driver dependent. Check your
database driver documentation for which of the five syntax styles,
described in PEP 249's paramstyle, is supported.
parse_dates:
- List of column names to parse as dates.
- Dict of ``{column_name: format string}`` where format string is
strftime compatible in case of parsing string times, or is one of
(D, s, ns, ms, us) in case of parsing integer timestamps.
- Dict of ``{column_name: arg dict}``, where the arg dict corresponds
to the keyword arguments of :func:`pandas.to_datetime`
Especially useful with databases without native Datetime support,
such as SQLite.
columns: List of column names to select from SQL table (only used when reading a table).
chunksize: If specified, return an iterator where `chunksize` is the number of rows to include in each chunk.
partition_column: column used to share the data between the workers (MUST be a INTEGER column)
lower_bound: the minimum value to be requested from the partition_column
upper_bound: the maximum value to be requested from the partition_column
Returns:
Pandas Dataframe
"""
from .sql import is_distributed, get_query_info
if not is_distributed(partition_column, lower_bound, upper_bound):
# Change this so that when `PandasOnRayIO` has a parallel `read_sql` we can
# still use it.
return PandasOnRayIO.read_sql(
sql,
con,
index_col,
coerce_float,
params,
parse_dates,
columns,
chunksize,
)
# starts the distributed alternative
cols_names, query = get_query_info(sql, con, partition_column)
num_parts = cls.block_partitions_cls._compute_num_partitions()
num_splits = min(len(cols_names), num_parts)
diff = (upper_bound - lower_bound) + 1
min_size = diff // num_parts
rest = diff % num_parts
partition_ids = []
index_ids = []
end = lower_bound - 1
for part in range(num_parts):
if rest:
size = min_size + 1
rest -= 1
else:
size = min_size
start = end + 1
end = start + size - 1
partition_id = _read_sql_with_offset_pandas_on_ray._remote(
args=(
partition_column,
start,
end,
num_splits,
query,
con,
index_col,
coerce_float,
params,
parse_dates,
columns,
chunksize,
),
num_return_vals=num_splits + 1,
)
partition_ids.append(
[PandasOnRayRemotePartition(obj) for obj in partition_id[:-1]])
index_ids.append(partition_id[-1])
new_index = pandas.RangeIndex(sum(ray.get(index_ids)))
new_query_compiler = cls.query_compiler_cls(
cls.block_partitions_cls(np.array(partition_ids)), new_index,
cols_names)
return new_query_compiler
def read_sql(
cls,
sql,
con,
index_col=None,
coerce_float=True,
params=None,
parse_dates=None,
columns=None,
chunksize=None,
partition_column=None,
lower_bound=None,
upper_bound=None,
max_sessions=None,
):
"""
Read SQL query or database table into a DataFrame.
Parameters
----------
sql : str or SQLAlchemy Selectable (select or text object)
SQL query to be executed or a table name.
con : SQLAlchemy connectable or str
Connection to database (sqlite3 connections are not supported).
index_col : str or list of str, optional
Column(s) to set as index(MultiIndex).
coerce_float : bool, default: True
Attempts to convert values of non-string, non-numeric objects
(like decimal.Decimal) to floating point, useful for SQL result sets.
params : list, tuple or dict, optional
List of parameters to pass to ``execute`` method. The syntax used
to pass parameters is database driver dependent. Check your
database driver documentation for which of the five syntax styles,
described in PEP 249's paramstyle, is supported.
parse_dates : list or dict, optional
The behavior is as follows:
- List of column names to parse as dates.
- Dict of `{column_name: format string}` where format string is
strftime compatible in case of parsing string times, or is one of
(D, s, ns, ms, us) in case of parsing integer timestamps.
- Dict of `{column_name: arg dict}`, where the arg dict corresponds
to the keyword arguments of ``pandas.to_datetime``.
Especially useful with databases without native Datetime support,
such as SQLite.
columns : list, optional
List of column names to select from SQL table (only used when reading a
table).
chunksize : int, optional
If specified, return an iterator where `chunksize` is the number of rows
to include in each chunk.
partition_column : str, optional
Column name used for data partitioning between the workers
(MUST be an INTEGER column).
lower_bound : int, optional
The minimum value to be requested from the `partition_column`.
upper_bound : int, optional
The maximum value to be requested from the `partition_column`.
max_sessions : int, optional
The maximum number of simultaneous connections allowed to use.
Returns
-------
BaseQueryCompiler
A new query compiler with imported data for further processing.
"""
from .sql import is_distributed, get_query_info
if not is_distributed(partition_column, lower_bound, upper_bound):
warnings.warn("Defaulting to Modin core implementation")
return PandasOnRayIO.read_sql(
sql,
con,
index_col,
coerce_float=coerce_float,
params=params,
parse_dates=parse_dates,
columns=columns,
chunksize=chunksize,
)
# starts the distributed alternative
cols_names, query = get_query_info(sql, con, partition_column)
num_parts = min(NPartitions.get(), max_sessions if max_sessions else 1)
num_splits = min(len(cols_names), num_parts)
diff = (upper_bound - lower_bound) + 1
min_size = diff // num_parts
rest = diff % num_parts
partition_ids = []
index_ids = []
end = lower_bound - 1
for part in range(num_parts):
if rest:
size = min_size + 1
rest -= 1
else:
size = min_size
start = end + 1
end = start + size - 1
partition_id = _read_sql_with_offset_pandas_on_ray.options(
num_returns=num_splits + 1).remote(
partition_column,
start,
end,
num_splits,
query,
con,
index_col,
coerce_float,
params,
parse_dates,
columns,
chunksize,
)
partition_ids.append(
[PandasOnRayFramePartition(obj) for obj in partition_id[:-1]])
index_ids.append(partition_id[-1])
new_index = pandas.RangeIndex(sum(ray.get(index_ids)))
new_query_compiler = cls.query_compiler_cls(
cls.frame_cls(np.array(partition_ids), new_index, cols_names))
new_query_compiler._modin_frame.synchronize_labels(axis=0)
return new_query_compiler
