def test_pandas_groupby_apply():
"""Test pandas.DataFrame.groupby(...).progress_apply"""
try:
from numpy.random import randint, rand
import pandas as pd
except ImportError:
raise SkipTest
with closing(StringIO()) as our_file:
tqdm.pandas(file=our_file, leave=False, ascii=True)
df = pd.DataFrame(randint(0, 50, (500, 3)))
df.groupby(0).progress_apply(lambda x: None)
dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc'))
dfs.groupby(['a']).progress_apply(lambda x: None)
df2 = df = pd.DataFrame(dict(a=randint(1, 8, 10000), b=rand(10000)))
res1 = df2.groupby("a").apply(max)
res2 = df2.groupby("a").progress_apply(max)
assert res1.equals(res2)
our_file.seek(0)
# don't expect final output since no `leave` and
# high dynamic `miniters`
nexres = '100%|##########|'
if nexres in our_file.read():
our_file.seek(0)
raise AssertionError("\nDid not expect:\n{0}\nIn:{1}\n".format(
nexres, our_file.read()))
with closing(StringIO()) as our_file:
tqdm.pandas(file=our_file, leave=True, ascii=True)
dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc'))
dfs.loc[0] = [2, 1, 1]
dfs['d'] = 100
expects = ['500/500', '1/1', '4/4', '2/2']
dfs.groupby(dfs.index).progress_apply(lambda x: None)
dfs.groupby('d').progress_apply(lambda x: None)
dfs.groupby(dfs.columns, axis=1).progress_apply(lambda x: None)
dfs.groupby([2, 2, 1, 1], axis=1).progress_apply(lambda x: None)
our_file.seek(0)
if our_file.read().count('100%') < 4:
our_file.seek(0)
raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format(
'100% at least four times', our_file.read()))
for exres in expects:
our_file.seek(0)
if our_file.getvalue().count(exres) < 1:
our_file.seek(0)
raise AssertionError(
"\nExpected:\n{0}\nIn:\n {1}\n".format(
exres + " at least once.", our_file.read()))
def test_main():
"""Test command line pipes"""
ls_out = _sh('ls').replace('\r\n', '\n')
ls = subprocess.Popen('ls',
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
res = _sh(sys.executable,
'-c',
'from tqdm.cli import main; main()',
stdin=ls.stdout,
stderr=subprocess.STDOUT)
ls.wait()
# actual test:
assert ls_out in res.replace('\r\n', '\n')
# semi-fake test which gets coverage:
_SYS = sys.stdin, sys.argv
with closing(StringIO()) as sys.stdin:
sys.argv = [
'', '--desc', 'Test CLI --delim', '--ascii', 'True', '--delim',
r'\0', '--buf_size', '64'
]
sys.stdin.write('\0'.join(map(str, _range(int(123)))))
# sys.stdin.write(b'\xff') # TODO
sys.stdin.seek(0)
main()
sys.stdin = IN_DATA_LIST
sys.argv = [
'', '--desc', 'Test CLI pipes', '--ascii', 'True', '--unit_scale',
'True'
]
import tqdm.__main__ # NOQA
with closing(StringIO()) as sys.stdin:
IN_DATA = '\0'.join(IN_DATA_LIST)
sys.stdin.write(IN_DATA)
sys.stdin.seek(0)
sys.argv = ['', '--ascii', '--bytes=True', '--unit_scale', 'False']
with closing(UnicodeIO()) as fp:
main(fp=fp)
assert str(len(IN_DATA)) in fp.getvalue()
sys.stdin = IN_DATA_LIST
# test --log
with closing(StringIO()) as sys.stdin:
sys.stdin.write('\0'.join(map(str, _range(int(123)))))
sys.stdin.seek(0)
# with closing(UnicodeIO()) as fp:
main(argv=['--log', 'DEBUG'], fp=NULL)
# assert "DEBUG:" in sys.stdout.getvalue()
sys.stdin = IN_DATA_LIST
# clean up
sys.stdin, sys.argv = _SYS
def test_enumerate():
"""Test contrib.tenumerate"""
with closing(StringIO()) as our_file:
a = range(9)
assert list(tenumerate(a, file=our_file)) == list(enumerate(a))
assert list(tenumerate(a, 42, file=our_file)) == list(enumerate(a, 42))
with closing(StringIO()) as our_file:
_ = list(tenumerate((i for i in a), file=our_file))
assert "100%" not in our_file.getvalue()
with closing(StringIO()) as our_file:
_ = list(tenumerate((i for i in a), file=our_file, total=len(a)))
assert "100%" in our_file.getvalue()
def test_pandas_groupby_apply():
"""Test pandas.DataFrame.groupby(...).progress_apply"""
try:
from numpy.random import randint
import pandas as pd
except:
raise SkipTest
with closing(StringIO()) as our_file:
tqdm.pandas(file=our_file, leave=False, ascii=True)
df = pd.DataFrame(randint(0, 50, (500, 3)))
df.groupby(0).progress_apply(lambda x: None)
dfs = pd.DataFrame(randint(0, 50, (500, 3)),
columns=list('abc'))
dfs.groupby(['a']).progress_apply(lambda x: None)
our_file.seek(0)
# don't expect final output since no `leave` and
# high dynamic `miniters`
nexres = '100%|##########|'
if nexres in our_file.read():
our_file.seek(0)
raise AssertionError("\nDid not expect:\n{0}\nIn:{1}\n".format(
nexres, our_file.read()))
def test_pandas_rolling_expanding():
"""Test pandas.(Series|DataFrame).(rolling|expanding)"""
try:
from numpy.random import randint
import pandas as pd
except ImportError:
raise SkipTest
with closing(StringIO()) as our_file:
tqdm.pandas(file=our_file, leave=True, ascii=True)
series = pd.Series(randint(0, 50, (123, )))
res1 = series.rolling(10).progress_apply(lambda x: 1, raw=True)
res2 = series.rolling(10).apply(lambda x: 1, raw=True)
assert res1.equals(res2)
res3 = series.expanding(10).progress_apply(lambda x: 2, raw=True)
res4 = series.expanding(10).apply(lambda x: 2, raw=True)
assert res3.equals(res4)
expects = ['114it'] # 123-10+1
for exres in expects:
our_file.seek(0)
if our_file.getvalue().count(exres) < 2:
our_file.seek(0)
raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format(
exres + " at least twice.", our_file.read()))
def test_pandas_series():
"""Test pandas.Series.progress_apply and .progress_map"""
try:
from numpy.random import randint
import pandas as pd
except ImportError:
raise SkipTest
with closing(StringIO()) as our_file:
tqdm.pandas(file=our_file, leave=True, ascii=True)
series = pd.Series(randint(0, 50, (123, )))
res1 = series.progress_apply(lambda x: x + 10)
res2 = series.apply(lambda x: x + 10)
assert res1.equals(res2)
res3 = series.progress_map(lambda x: x + 10)
res4 = series.map(lambda x: x + 10)
assert res3.equals(res4)
expects = ['100%', '123/123']
for exres in expects:
our_file.seek(0)
if our_file.getvalue().count(exres) < 2:
our_file.seek(0)
raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format(
exres + " at least twice.", our_file.read()))
def incr_bar(x):
with closing(StringIO()) as our_file:
for _ in trange(
total, file=our_file,
lock_args=None if blocking else (False,),
miniters=1, mininterval=0, maxinterval=0):
pass
return x + 1
def test_process_map():
"""Test contrib.concurrent.process_map"""
with closing(StringIO()) as our_file:
a = range(9)
b = [i + 1 for i in a]
try:
assert process_map(incr, a, file=our_file) == b
except ImportError:
raise SkipTest
def test_enumerate_numpy():
"""Test contrib.tenumerate(numpy.ndarray)"""
try:
import numpy as np
except ImportError:
raise SkipTest
with closing(StringIO()) as our_file:
a = np.random.random((42, 1337))
assert list(tenumerate(a, file=our_file)) == list(np.ndenumerate(a))
async def test_nested():
"""Test asyncio nested"""
with closing(StringIO()) as our_file:
async for row in tqdm(trange(9, desc="inner", file=our_file),
desc="outer",
file=our_file):
pass
assert 'inner: 100%' in our_file.getvalue()
assert 'outer: 100%' in our_file.getvalue()
def test_product():
"""Test contrib.itertools.product"""
with closing(StringIO()) as our_file:
a = range(9)
assert list(product(a, a[::-1], file=our_file)) == \
list(itertools.product(a, a[::-1]))
assert list(product(a, NoLenIter(a), file=our_file)) == \
list(itertools.product(a, NoLenIter(a)))
def test_monitoring_and_cleanup():
"""Test for stalled tqdm instance and monitor deletion"""
# Note: should fix miniters for these tests, else with dynamic_miniters
# it's too complicated to handle with monitoring update and maxinterval...
maxinterval = 2
total = 1000
# Setup a discrete timer
timer = DiscreteTimer()
# And a fake sleeper
sleeper = FakeSleep(timer)
# Setup TMonitor to use the timer
TMonitor._time = timer.time
TMonitor._event = make_create_fake_sleep_event(sleeper.sleep)
# Set monitor interval
tqdm.monitor_interval = maxinterval
with closing(StringIO()) as our_file:
with tqdm(total=total,
file=our_file,
miniters=500,
mininterval=0.1,
maxinterval=maxinterval) as t:
cpu_timify(t, timer)
# Do a lot of iterations in a small timeframe
# (smaller than monitor interval)
timer.sleep(maxinterval / 2) # monitor won't wake up
t.update(500)
# check that our fixed miniters is still there
assert t.miniters == 500
# Then do 1 it after monitor interval, so that monitor kicks in
timer.sleep(maxinterval * 2)
t.update(1)
# Wait for the monitor to get out of sleep's loop and update tqdm..
timeend = timer.time()
while not (t.monitor.woken >= timeend and t.miniters == 1):
timer.sleep(1) # Force monitor to wake up if it woken too soon
sleep(0.000001) # sleep to allow interrupt (instead of pass)
assert t.miniters == 1 # check that monitor corrected miniters
# Note: at this point, there may be a race condition: monitor saved
# current woken time but timer.sleep() happen just before monitor
# sleep. To fix that, either sleep here or increase time in a loop
# to ensure that monitor wakes up at some point.
# Try again but already at miniters = 1 so nothing will be done
timer.sleep(maxinterval * 2)
t.update(2)
timeend = timer.time()
while not (t.monitor.woken >= timeend):
timer.sleep(1) # Force monitor to wake up if it woken too soon
sleep(0.000001)
# Wait for the monitor to get out of sleep's loop and update tqdm..
assert t.miniters == 1 # check that monitor corrected miniters
# Check that class var monitor is deleted if no instance left
tqdm.monitor_interval = 10
assert tqdm.monitor is None
def test_thread_map():
"""Test contrib.concurrent.thread_map"""
with closing(StringIO()) as our_file:
a = range(9)
b = [i + 1 for i in a]
try:
assert thread_map(lambda x: x + 1, a, file=our_file) == b
except ImportError:
raise SkipTest
assert thread_map(incr, a, file=our_file) == b
async def test_as_completed():
"""Test asyncio as_completed"""
with closing(StringIO()) as our_file:
t = time()
skew = time() - t
for i in as_completed([asyncio.sleep(0.01) for _ in range(100)],
file=our_file):
await i
assert time() - t - 2 * skew < (0.01 * 100) / 2, "Assuming >= 2 cores"
assert '100/100' in our_file.getvalue()
async def test_coroutines():
"""Test asyncio coroutine.send"""
with closing(StringIO()) as our_file:
with tqdm(count(), file=our_file) as pbar:
async for row in pbar:
if row == 9:
pbar.send(-10)
elif row < 0:
assert row == -9
break
assert '10it' in our_file.getvalue()
async def test_as_completed():
"""Test asyncio as_completed"""
with closing(StringIO()) as our_file:
t = time()
skew = time() - t
for i in as_completed(
[asyncio.sleep(0.01 * i) for i in range(30, 0, -1)],
file=our_file):
await i
assert 0.29 < time() - t - 2 * skew < 0.31
assert '30/30' in our_file.getvalue()
def test_map():
"""Test contrib.tmap"""
with closing(StringIO()) as our_file:
a = range(9)
b = [i + 1 for i in a]
if sys.version_info[:1] < (3, ):
assert tmap(lambda x: x + 1, a, file=our_file) == map(incr, a)
else:
gen = tmap(lambda x: x + 1, a, file=our_file)
assert gen != b
assert list(gen) == b
def test_zip():
"""Test contrib.tzip"""
with closing(StringIO()) as our_file:
a = range(9)
b = [i + 1 for i in a]
if sys.version_info[:1] < (3, ):
assert tzip(a, b, file=our_file) == zip(a, b)
else:
gen = tzip(a, b, file=our_file)
assert gen != list(zip(a, b))
assert list(gen) == list(zip(a, b))
async def test_range():
"""Test asyncio range"""
with closing(StringIO()) as our_file:
async for row in tqdm(range(9), desc="range", file=our_file):
pass
assert '9/9' in our_file.getvalue()
our_file.seek(0)
our_file.truncate()
async for row in trange(9, desc="trange", file=our_file):
pass
assert '9/9' in our_file.getvalue()
def test_monitoring_multi():
"""Test on multiple bars, one not needing miniters adjustment"""
# Note: should fix miniters for these tests, else with dynamic_miniters
# it's too complicated to handle with monitoring update and maxinterval...
maxinterval = 2
total = 1000
# Setup a discrete timer
timer = DiscreteTimer()
# And a fake sleeper
sleeper = FakeSleep(timer)
# Setup TMonitor to use the timer
TMonitor._time = timer.time
TMonitor._event = make_create_fake_sleep_event(sleeper.sleep)
# Set monitor interval
tqdm.monitor_interval = maxinterval
with closing(StringIO()) as our_file:
with tqdm(total=total,
file=our_file,
miniters=500,
mininterval=0.1,
maxinterval=maxinterval) as t1:
# Set high maxinterval for t2 so monitor does not need to adjust it
with tqdm(total=total,
file=our_file,
miniters=500,
mininterval=0.1,
maxinterval=1E5) as t2:
cpu_timify(t1, timer)
cpu_timify(t2, timer)
# Do a lot of iterations in a small timeframe
timer.sleep(maxinterval / 2)
t1.update(500)
t2.update(500)
assert t1.miniters == 500
assert t2.miniters == 500
# Then do 1 it after monitor interval, so that monitor kicks in
timer.sleep(maxinterval * 2)
t1.update(1)
t2.update(1)
# Wait for the monitor to get out of sleep and update tqdm
timeend = timer.time()
while not (t1.monitor.woken >= timeend and t1.miniters == 1):
timer.sleep(1)
sleep(0.000001)
assert t1.miniters == 1 # check that monitor corrected miniters
assert t2.miniters == 500 # check that t2 was not adjusted
# Check that class var monitor is deleted if no instance left
tqdm.monitor_interval = 10
assert tqdm.monitor is None
def test_pandas_setup():
"""Test tqdm.pandas()"""
try:
from numpy.random import randint
import pandas as pd
except ImportError:
raise SkipTest
with closing(StringIO()) as our_file:
tqdm.pandas(file=our_file, leave=True, ascii=True, total=123)
series = pd.Series(randint(0, 50, (100, )))
series.progress_apply(lambda x: x + 10)
res = our_file.getvalue()
assert '100/123' in res
async def test_generators():
"""Test asyncio generators"""
with closing(StringIO()) as our_file:
async for row in tqdm(count(), desc="counter", file=our_file):
if row >= 8:
break
assert '9it' in our_file.getvalue()
our_file.seek(0)
our_file.truncate()
async for row in tqdm(acount(), desc="async_counter", file=our_file):
if row >= 8:
break
assert '9it' in our_file.getvalue()
def test_pandas_deprecation():
"""Test bar object instance as argument deprecation"""
try:
from numpy.random import randint
from tqdm import tqdm_pandas
import pandas as pd
except ImportError:
raise SkipTest
with closing(StringIO()) as our_file:
tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20))
df = pd.DataFrame(randint(0, 50, (500, 3)))
df.groupby(0).progress_apply(lambda x: None)
# Check deprecation message
assert "TqdmDeprecationWarning" in our_file.getvalue()
assert "instead of `tqdm_pandas(tqdm(...))`" in our_file.getvalue()
with closing(StringIO()) as our_file:
tqdm_pandas(tqdm, file=our_file, leave=False, ascii=True, ncols=20)
df = pd.DataFrame(randint(0, 50, (500, 3)))
df.groupby(0).progress_apply(lambda x: None)
# Check deprecation message
assert "TqdmDeprecationWarning" in our_file.getvalue()
assert "instead of `tqdm_pandas(tqdm, ...)`" in our_file.getvalue()
def test_pandas_map():
"""Test pandas.Series.progress_map"""
try:
from numpy.random import randint
import pandas as pd
except:
raise SkipTest
with closing(StringIO()) as our_file:
tqdm.pandas(file=our_file, leave=True, ascii=True)
dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc'))
dfs.a.progress_map(lambda x: None)
if our_file.getvalue().count('100%') < 1:
raise AssertionError("\nExpected:\n{0}\nIn:{1}\n".format(
'100% at least twice', our_file.getvalue()))
def test_pandas_data_frame():
"""Test pandas.DataFrame.progress_apply and .progress_applymap"""
try:
from numpy.random import randint
import pandas as pd
except ImportError:
raise SkipTest
with closing(StringIO()) as our_file:
tqdm.pandas(file=our_file, leave=True, ascii=True)
df = pd.DataFrame(randint(0, 50, (100, 200)))
def task_func(x):
return x + 1
# applymap
res1 = df.progress_applymap(task_func)
res2 = df.applymap(task_func)
assert res1.equals(res2)
# apply unhashable
res1 = []
df.progress_apply(res1.extend)
assert len(res1) == df.size
# apply
for axis in [0, 1, 'index', 'columns']:
res3 = df.progress_apply(task_func, axis=axis)
res4 = df.apply(task_func, axis=axis)
assert res3.equals(res4)
our_file.seek(0)
if our_file.read().count('100%') < 3:
our_file.seek(0)
raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format(
'100% at least three times', our_file.read()))
# apply_map, apply axis=0, apply axis=1
expects = ['20000/20000', '200/200', '100/100']
for exres in expects:
our_file.seek(0)
if our_file.getvalue().count(exres) < 1:
our_file.seek(0)
raise AssertionError(
"\nExpected:\n{0}\nIn:\n {1}\n".format(
exres + " at least once.", our_file.read()))
def test_pandas_apply_args_deprecation():
"""Test warning info in
`pandas.Dataframe(Series).progress_apply(func, *args)`"""
try:
from numpy.random import randint
from tqdm import tqdm_pandas
import pandas as pd
except ImportError:
raise SkipTest
with closing(StringIO()) as our_file:
tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20))
df = pd.DataFrame(randint(0, 50, (500, 3)))
df.progress_apply(lambda x: None, 1) # 1 shall cause a warning
# Check deprecation message
res = our_file.getvalue()
assert all([i in res for i in (
"TqdmDeprecationWarning", "not supported",
"keyword arguments instead")])
def test_monitoring_and_cleanup():
"""Test for stalled tqdm instance and monitor deletion"""
# Note: should fix miniters for these tests, else with dynamic_miniters
# it's too complicated to handle with monitoring update and maxinterval...
maxinterval = tqdm.monitor_interval
assert maxinterval == 10
total = 1000
with closing(StringIO()) as our_file:
with tqdm(total=total,
file=our_file,
miniters=500,
mininterval=0.1,
maxinterval=maxinterval) as t:
cpu_timify(t, Time)
# Do a lot of iterations in a small timeframe
# (smaller than monitor interval)
Time.fake_sleep(maxinterval / 10) # monitor won't wake up
t.update(500)
# check that our fixed miniters is still there
assert t.miniters <= 500 # TODO: should really be == 500
# Then do 1 it after monitor interval, so that monitor kicks in
Time.fake_sleep(maxinterval)
t.update(1)
# Wait for the monitor to get out of sleep's loop and update tqdm..
timeend = Time.time()
while not (t.monitor.woken >= timeend and t.miniters == 1):
Time.fake_sleep(1) # Force awake up if it woken too soon
assert t.miniters == 1 # check that monitor corrected miniters
# Note: at this point, there may be a race condition: monitor saved
# current woken time but Time.sleep() happen just before monitor
# sleep. To fix that, either sleep here or increase time in a loop
# to ensure that monitor wakes up at some point.
# Try again but already at miniters = 1 so nothing will be done
Time.fake_sleep(maxinterval)
t.update(2)
timeend = Time.time()
while t.monitor.woken < timeend:
Time.fake_sleep(1) # Force awake if it woken too soon
# Wait for the monitor to get out of sleep's loop and update tqdm
assert t.miniters == 1 # check that monitor corrected miniters
def test_pandas_leave():
"""Test pandas with `leave=True`"""
try:
from numpy.random import randint
import pandas as pd
except ImportError:
raise SkipTest
with closing(StringIO()) as our_file:
df = pd.DataFrame(randint(0, 100, (1000, 6)))
tqdm.pandas(file=our_file, leave=True, ascii=True)
df.groupby(0).progress_apply(lambda x: None)
our_file.seek(0)
exres = '100%|##########| 100/100'
if exres not in our_file.read():
our_file.seek(0)
raise AssertionError("\nExpected:\n{0}\nIn:{1}\n".format(
exres, our_file.read()))
def test_pandas_apply():
""" Test pandas.DataFrame.progress_apply """
try:
from numpy.random import randint
from tqdm import tqdm_pandas
import pandas as pd
except:
raise SkipTest
with closing(StringIO()) as our_file:
df = pd.DataFrame(randint(0, 100, (1000, 6)))
tqdm_pandas(tqdm(file=our_file, leave=True, ascii=True))
df.progress_apply(lambda x: None)
our_file.seek(0)
if '/6' not in our_file.read():
our_file.seek(0)
raise AssertionError("\nExpected:\n{0}\nIn:{1}\n".format(
'/6', our_file.read()))
def test_monitoring_multi():
"""Test on multiple bars, one not needing miniters adjustment"""
# Note: should fix miniters for these tests, else with dynamic_miniters
# it's too complicated to handle with monitoring update and maxinterval...
maxinterval = tqdm.monitor_interval
assert maxinterval == 10
total = 1000
with closing(StringIO()) as our_file:
with tqdm(total=total,
file=our_file,
miniters=500,
mininterval=0.1,
maxinterval=maxinterval) as t1:
# Set high maxinterval for t2 so monitor does not need to adjust it
with tqdm(total=total,
file=our_file,
miniters=500,
mininterval=0.1,
maxinterval=1E5) as t2:
cpu_timify(t1, Time)
cpu_timify(t2, Time)
# Do a lot of iterations in a small timeframe
Time.fake_sleep(maxinterval / 10)
t1.update(500)
t2.update(500)
assert t1.miniters <= 500 # TODO: should really be == 500
assert t2.miniters == 500
# Then do 1 it after monitor interval, so that monitor kicks in
Time.fake_sleep(maxinterval)
t1.update(1)
t2.update(1)
# Wait for the monitor to get out of sleep and update tqdm
timeend = Time.time()
while not (t1.monitor.woken >= timeend and t1.miniters == 1):
Time.fake_sleep(1)
assert t1.miniters == 1 # check that monitor corrected miniters
assert t2.miniters == 500 # check that t2 was not adjusted