def from_pandas(cls, df, return_dims=False):
"""Return the partitions from Pandas DataFrame."""
def update_bar(pbar, f):
if ProgressBar.get():
pbar.update(1)
return f
num_splits = NPartitions.get()
put_func = cls._partition_class.put
row_chunksize, col_chunksize = compute_chunksize(df, num_splits)
bar_format = (
"{l_bar}{bar}{r_bar}"
if os.environ.get("DEBUG_PROGRESS_BAR", "False") == "True" else
"{desc}: {percentage:3.0f}%{bar} Elapsed time: {elapsed}, estimated remaining time: {remaining}"
)
if ProgressBar.get():
with warnings.catch_warnings():
warnings.simplefilter("ignore")
try:
from tqdm.autonotebook import tqdm as tqdm_notebook
except ImportError:
raise ImportError(
"Please pip install tqdm to use the progress bar")
rows = max(1, round(len(df) / row_chunksize))
cols = max(1, round(len(df.columns) / col_chunksize))
update_count = rows * cols
pbar = tqdm_notebook(
total=round(update_count),
desc="Distributing Dataframe",
bar_format=bar_format,
)
else:
pbar = None
parts = [[
update_bar(
pbar,
put_func(df.iloc[i:i + row_chunksize,
j:j + col_chunksize].copy()),
) for j in range(0, len(df.columns), col_chunksize)
] for i in range(0, len(df), row_chunksize)]
if ProgressBar.get():
pbar.close()
if not return_dims:
return np.array(parts)
else:
row_lengths = [
row_chunksize if i + row_chunksize < len(df) else
len(df) % row_chunksize or row_chunksize
for i in range(0, len(df), row_chunksize)
]
col_widths = [
col_chunksize if i + col_chunksize < len(df.columns) else
len(df.columns) % col_chunksize or col_chunksize
for i in range(0, len(df.columns), col_chunksize)
]
return np.array(parts), row_lengths, col_widths
def magic(*args, **kwargs):
result_parts = f(*args, **kwargs)
if ProgressBar.get():
current_frame = inspect.currentframe()
function_name = None
while function_name != "<module>":
(
filename,
line_number,
function_name,
lines,
index,
) = inspect.getframeinfo(current_frame)
current_frame = current_frame.f_back
t = threading.Thread(
target=call_progress_bar,
args=(result_parts, line_number),
)
t.start()
# We need to know whether or not we are in a jupyter notebook
from IPython import get_ipython
try:
ipy_str = str(type(get_ipython()))
if "zmqshell" not in ipy_str:
t.join()
except Exception:
pass
return result_parts
def update_bar(pbar, f):
if ProgressBar.get():
pbar.update(1)
return f