def query_iterator(): offset = 0 while True: kwargs["sql"] = "SELECT * FROM ({}) LIMIT {} OFFSET {}".format(sql, chunksize, offset) offset += chunksize df = DataFrame(query_compiler=BaseFactory.read_sql(**kwargs)) if df.empty: break else: yield df
def read_sql( sql, con, index_col=None, coerce_float=True, params=None, parse_dates=None, columns=None, chunksize=None, ): """ Read SQL query or database table into a DataFrame. Args: sql: string or SQLAlchemy Selectable (select or text object) SQL query to be executed or a table name. con: SQLAlchemy connectable (engine/connection) or database string URI or DBAPI2 connection (fallback mode) index_col: Column(s) to set as index(MultiIndex). coerce_float: Attempts to convert values of non-string, non-numeric objects (like decimal.Decimal) to floating point, useful for SQL result sets. params: List of parameters to pass to execute method. The syntax used to pass parameters is database driver dependent. Check your database driver documentation for which of the five syntax styles, described in PEP 249's paramstyle, is supported. parse_dates: - List of column names to parse as dates. - Dict of ``{column_name: format string}`` where format string is strftime compatible in case of parsing string times, or is one of (D, s, ns, ms, us) in case of parsing integer timestamps. - Dict of ``{column_name: arg dict}``, where the arg dict corresponds to the keyword arguments of :func:`pandas.to_datetime` Especially useful with databases without native Datetime support, such as SQLite. columns: List of column names to select from SQL table (only used when reading a table). chunksize: If specified, return an iterator where `chunksize` is the number of rows to include in each chunk. Returns: Modin Dataframe """ _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) if kwargs.get("chunksize", None) is not None: kwargs["chunksize"] = None def query_iterator(): offset = 0 while True: kwargs["sql"] = "SELECT * FROM ({}) LIMIT {} OFFSET {}".format(sql, chunksize, offset) offset += chunksize df = DataFrame(query_compiler=BaseFactory.read_sql(**kwargs)) if df.empty: break else: yield df return query_iterator() return DataFrame(query_compiler=BaseFactory.read_sql(**kwargs))
def read_sql( sql, con, index_col=None, coerce_float=True, params=None, parse_dates=None, columns=None, chunksize=None, ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) return DataFrame(query_compiler=BaseFactory.read_sql(**kwargs))
def read_sql( sql, con, index_col=None, coerce_float=True, params=None, parse_dates=None, columns=None, chunksize=None, ): """ Read SQL query or database table into a DataFrame. Args: sql: string or SQLAlchemy Selectable (select or text object) SQL query to be executed or a table name. con: SQLAlchemy connectable (engine/connection) or database string URI or DBAPI2 connection (fallback mode) index_col: Column(s) to set as index(MultiIndex). coerce_float: Attempts to convert values of non-string, non-numeric objects (like decimal.Decimal) to floating point, useful for SQL result sets. params: List of parameters to pass to execute method. The syntax used to pass parameters is database driver dependent. Check your database driver documentation for which of the five syntax styles, described in PEP 249's paramstyle, is supported. parse_dates: - List of column names to parse as dates. - Dict of ``{column_name: format string}`` where format string is strftime compatible in case of parsing string times, or is one of (D, s, ns, ms, us) in case of parsing integer timestamps. - Dict of ``{column_name: arg dict}``, where the arg dict corresponds to the keyword arguments of :func:`pandas.to_datetime` Especially useful with databases without native Datetime support, such as SQLite. columns: List of column names to select from SQL table (only used when reading a table). chunksize: If specified, return an iterator where `chunksize` is the number of rows to include in each chunk. Returns: Modin Dataframe """ _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) from modin.data_management.factories import BaseFactory if kwargs.get("chunksize") is not None: ErrorMessage.default_to_pandas("Parameters provided [chunksize]") df_gen = pandas.read_sql(**kwargs) return (DataFrame(query_compiler=BaseFactory.from_pandas(df)) for df in df_gen) return DataFrame(query_compiler=BaseFactory.read_sql(**kwargs))