def preprocess(df):
"""Preprocess the dataframe
- Appends the index to the dataframe when it contains information
- Rename the "index" column to "df_index", if exists
- Convert the DataFrame's columns to str
Args:
df: the pandas DataFrame
Returns:
The preprocessed DataFrame
"""
# Treat index as any other column
if (
not pd.Index(np.arange(0, len(df))).equals(df.index)
or df.index.dtype != np.int64
):
df = df.reset_index()
# Rename reserved column names
df = rename_index(df)
# Ensure that columns are strings
df.columns = df.columns.astype("str")
return df
def __init__(self, df, config_file: Path = None, **kwargs):
if config_file:
config.config.set_file(str(config_file))
config.set_kwargs(kwargs)
# Treat index as any other column
if (not pd.Index(np.arange(0, len(df))).equals(df.index)
or df.index.dtype != np.int64):
df = df.reset_index()
# Rename reserved column names
df = rename_index(df)
# Remove spaces and colons from column names
df = clean_column_names(df)
# Sort column names
sort = config["sort"].get(str)
if sys.version_info[1] <= 5 and sort != "None":
warnings.warn("Sorting is supported from Python 3.6+")
if sort in ["asc", "ascending"]:
df = df.reindex(sorted(df.columns, key=lambda s: s.casefold()),
axis=1)
elif sort in ["desc", "descending"]:
df = df.reindex(reversed(
sorted(df.columns, key=lambda s: s.casefold())),
axis=1)
elif sort != "None":
raise ValueError(
'"sort" should be "ascending", "descending" or None.')
# Store column order
config["column_order"] = df.columns.tolist()
# Get dataset statistics
description_set = describe_df(df)
# Get sample
sample = {}
n_head = config["samples"]["head"].get(int)
if n_head > 0:
sample["head"] = df.head(n=n_head)
n_tail = config["samples"]["tail"].get(int)
if n_tail > 0:
sample["tail"] = df.tail(n=n_tail)
# Render HTML
self.html = to_html(sample, description_set)
self.minify_html = config["minify_html"].get(bool)
self.use_local_assets = config["use_local_assets"].get(bool)
self.title = config["title"].get(str)
self.description_set = description_set
self.sample = sample
def preprocess(df):
# Treat index as any other column
if (not pd.Index(np.arange(0, len(df))).equals(df.index)
or df.index.dtype != np.int64):
df = df.reset_index()
# Rename reserved column names
df = rename_index(df)
# Ensure that columns are strings
df.columns = df.columns.astype("str")
return df
def __init__(self, df, minimal=False, config_file: Path = None, **kwargs):
if sys.version_info <= (3, 5):
warnings.warn(
"This is the last release to support Python 3.5, please upgrade.",
category=DeprecationWarning,
)
if config_file is not None and minimal:
raise ValueError(
"Arguments `config_file` and `minimal` are mutually exclusive."
)
if minimal:
config_file = get_config_minimal()
if config_file:
config.set_file(str(config_file))
config.set_kwargs(kwargs)
self.date_start = datetime.utcnow()
# Treat index as any other column
if (not pd.Index(np.arange(0, len(df))).equals(df.index)
or df.index.dtype != np.int64):
df = df.reset_index()
# Rename reserved column names
df = rename_index(df)
# Ensure that columns are strings
df.columns = df.columns.astype("str")
# Get dataset statistics
description_set = describe_df(df)
# Build report structure
self.sample = self.get_sample(df)
self.title = config["title"].get(str)
self.description_set = description_set
self.date_end = datetime.utcnow()
disable_progress_bar = not config["progress_bar"].get(bool)
with tqdm(total=1,
desc="build report structure",
disable=disable_progress_bar) as pbar:
self.report = get_report_structure(self.date_start, self.date_end,
self.sample, description_set)
pbar.update()
def __init__(self, df, minimal=False, config_file: Path = None, **kwargs):
if config_file is not None and minimal:
raise ValueError(
"Arguments `config_file` and `minimal` are mutually exclusive."
)
if minimal:
config_file = get_config_minimal()
if config_file:
config.config.set_file(str(config_file))
config.set_kwargs(kwargs)
self.date_start = datetime.utcnow()
# Treat index as any other column
if (
not pd.Index(np.arange(0, len(df))).equals(df.index)
or df.index.dtype != np.int64
):
df = df.reset_index()
# Rename reserved column names
df = rename_index(df)
# Ensure that columns are strings
df.columns = df.columns.astype("str")
# Sort names according to config (asc, desc, no sort)
df = self.sort_column_names(df)
config["column_order"] = df.columns.tolist()
# Get dataset statistics
description_set = describe_df(df)
# Build report structure
self.sample = self.get_sample(df)
self.title = config["title"].get(str)
self.description_set = description_set
self.date_end = datetime.utcnow()
self.report = get_report_structure(
self.date_start, self.date_end, self.sample, description_set
)
