def test_pandas_groupby_apply(): """Test pandas.DataFrame.groupby(...).progress_apply""" try: from numpy.random import randint, rand import pandas as pd except ImportError: raise SkipTest with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=False, ascii=True) df = pd.DataFrame(randint(0, 50, (500, 3))) df.groupby(0).progress_apply(lambda x: None) dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc')) dfs.groupby(['a']).progress_apply(lambda x: None) df2 = df = pd.DataFrame(dict(a=randint(1, 8, 10000), b=rand(10000))) res1 = df2.groupby("a").apply(max) res2 = df2.groupby("a").progress_apply(max) assert res1.equals(res2) our_file.seek(0) # don't expect final output since no `leave` and # high dynamic `miniters` nexres = '100%|##########|' if nexres in our_file.read(): our_file.seek(0) raise AssertionError("\nDid not expect:\n{0}\nIn:{1}\n".format( nexres, our_file.read())) with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True) dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc')) dfs.loc[0] = [2, 1, 1] dfs['d'] = 100 expects = ['500/500', '1/1', '4/4', '2/2'] dfs.groupby(dfs.index).progress_apply(lambda x: None) dfs.groupby('d').progress_apply(lambda x: None) dfs.groupby(dfs.columns, axis=1).progress_apply(lambda x: None) dfs.groupby([2, 2, 1, 1], axis=1).progress_apply(lambda x: None) our_file.seek(0) if our_file.read().count('100%') < 4: our_file.seek(0) raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( '100% at least four times', our_file.read())) for exres in expects: our_file.seek(0) if our_file.getvalue().count(exres) < 1: our_file.seek(0) raise AssertionError( "\nExpected:\n{0}\nIn:\n {1}\n".format( exres + " at least once.", our_file.read()))
def test_main(): """Test command line pipes""" ls_out = _sh('ls').replace('\r\n', '\n') ls = subprocess.Popen('ls', stdout=subprocess.PIPE, stderr=subprocess.STDOUT) res = _sh(sys.executable, '-c', 'from tqdm.cli import main; main()', stdin=ls.stdout, stderr=subprocess.STDOUT) ls.wait() # actual test: assert ls_out in res.replace('\r\n', '\n') # semi-fake test which gets coverage: _SYS = sys.stdin, sys.argv with closing(StringIO()) as sys.stdin: sys.argv = [ '', '--desc', 'Test CLI --delim', '--ascii', 'True', '--delim', r'\0', '--buf_size', '64' ] sys.stdin.write('\0'.join(map(str, _range(int(123))))) # sys.stdin.write(b'\xff') # TODO sys.stdin.seek(0) main() sys.stdin = IN_DATA_LIST sys.argv = [ '', '--desc', 'Test CLI pipes', '--ascii', 'True', '--unit_scale', 'True' ] import tqdm.__main__ # NOQA with closing(StringIO()) as sys.stdin: IN_DATA = '\0'.join(IN_DATA_LIST) sys.stdin.write(IN_DATA) sys.stdin.seek(0) sys.argv = ['', '--ascii', '--bytes=True', '--unit_scale', 'False'] with closing(UnicodeIO()) as fp: main(fp=fp) assert str(len(IN_DATA)) in fp.getvalue() sys.stdin = IN_DATA_LIST # test --log with closing(StringIO()) as sys.stdin: sys.stdin.write('\0'.join(map(str, _range(int(123))))) sys.stdin.seek(0) # with closing(UnicodeIO()) as fp: main(argv=['--log', 'DEBUG'], fp=NULL) # assert "DEBUG:" in sys.stdout.getvalue() sys.stdin = IN_DATA_LIST # clean up sys.stdin, sys.argv = _SYS
def test_enumerate(): """Test contrib.tenumerate""" with closing(StringIO()) as our_file: a = range(9) assert list(tenumerate(a, file=our_file)) == list(enumerate(a)) assert list(tenumerate(a, 42, file=our_file)) == list(enumerate(a, 42)) with closing(StringIO()) as our_file: _ = list(tenumerate((i for i in a), file=our_file)) assert "100%" not in our_file.getvalue() with closing(StringIO()) as our_file: _ = list(tenumerate((i for i in a), file=our_file, total=len(a))) assert "100%" in our_file.getvalue()
def test_pandas_groupby_apply(): """Test pandas.DataFrame.groupby(...).progress_apply""" try: from numpy.random import randint import pandas as pd except: raise SkipTest with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=False, ascii=True) df = pd.DataFrame(randint(0, 50, (500, 3))) df.groupby(0).progress_apply(lambda x: None) dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc')) dfs.groupby(['a']).progress_apply(lambda x: None) our_file.seek(0) # don't expect final output since no `leave` and # high dynamic `miniters` nexres = '100%|##########|' if nexres in our_file.read(): our_file.seek(0) raise AssertionError("\nDid not expect:\n{0}\nIn:{1}\n".format( nexres, our_file.read()))
def test_pandas_rolling_expanding(): """Test pandas.(Series|DataFrame).(rolling|expanding)""" try: from numpy.random import randint import pandas as pd except ImportError: raise SkipTest with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True) series = pd.Series(randint(0, 50, (123, ))) res1 = series.rolling(10).progress_apply(lambda x: 1, raw=True) res2 = series.rolling(10).apply(lambda x: 1, raw=True) assert res1.equals(res2) res3 = series.expanding(10).progress_apply(lambda x: 2, raw=True) res4 = series.expanding(10).apply(lambda x: 2, raw=True) assert res3.equals(res4) expects = ['114it'] # 123-10+1 for exres in expects: our_file.seek(0) if our_file.getvalue().count(exres) < 2: our_file.seek(0) raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( exres + " at least twice.", our_file.read()))
def test_pandas_series(): """Test pandas.Series.progress_apply and .progress_map""" try: from numpy.random import randint import pandas as pd except ImportError: raise SkipTest with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True) series = pd.Series(randint(0, 50, (123, ))) res1 = series.progress_apply(lambda x: x + 10) res2 = series.apply(lambda x: x + 10) assert res1.equals(res2) res3 = series.progress_map(lambda x: x + 10) res4 = series.map(lambda x: x + 10) assert res3.equals(res4) expects = ['100%', '123/123'] for exres in expects: our_file.seek(0) if our_file.getvalue().count(exres) < 2: our_file.seek(0) raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( exres + " at least twice.", our_file.read()))
def incr_bar(x): with closing(StringIO()) as our_file: for _ in trange( total, file=our_file, lock_args=None if blocking else (False,), miniters=1, mininterval=0, maxinterval=0): pass return x + 1
def test_process_map(): """Test contrib.concurrent.process_map""" with closing(StringIO()) as our_file: a = range(9) b = [i + 1 for i in a] try: assert process_map(incr, a, file=our_file) == b except ImportError: raise SkipTest
def test_enumerate_numpy(): """Test contrib.tenumerate(numpy.ndarray)""" try: import numpy as np except ImportError: raise SkipTest with closing(StringIO()) as our_file: a = np.random.random((42, 1337)) assert list(tenumerate(a, file=our_file)) == list(np.ndenumerate(a))
async def test_nested(): """Test asyncio nested""" with closing(StringIO()) as our_file: async for row in tqdm(trange(9, desc="inner", file=our_file), desc="outer", file=our_file): pass assert 'inner: 100%' in our_file.getvalue() assert 'outer: 100%' in our_file.getvalue()
def test_product(): """Test contrib.itertools.product""" with closing(StringIO()) as our_file: a = range(9) assert list(product(a, a[::-1], file=our_file)) == \ list(itertools.product(a, a[::-1])) assert list(product(a, NoLenIter(a), file=our_file)) == \ list(itertools.product(a, NoLenIter(a)))
def test_monitoring_and_cleanup(): """Test for stalled tqdm instance and monitor deletion""" # Note: should fix miniters for these tests, else with dynamic_miniters # it's too complicated to handle with monitoring update and maxinterval... maxinterval = 2 total = 1000 # Setup a discrete timer timer = DiscreteTimer() # And a fake sleeper sleeper = FakeSleep(timer) # Setup TMonitor to use the timer TMonitor._time = timer.time TMonitor._event = make_create_fake_sleep_event(sleeper.sleep) # Set monitor interval tqdm.monitor_interval = maxinterval with closing(StringIO()) as our_file: with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, maxinterval=maxinterval) as t: cpu_timify(t, timer) # Do a lot of iterations in a small timeframe # (smaller than monitor interval) timer.sleep(maxinterval / 2) # monitor won't wake up t.update(500) # check that our fixed miniters is still there assert t.miniters == 500 # Then do 1 it after monitor interval, so that monitor kicks in timer.sleep(maxinterval * 2) t.update(1) # Wait for the monitor to get out of sleep's loop and update tqdm.. timeend = timer.time() while not (t.monitor.woken >= timeend and t.miniters == 1): timer.sleep(1) # Force monitor to wake up if it woken too soon sleep(0.000001) # sleep to allow interrupt (instead of pass) assert t.miniters == 1 # check that monitor corrected miniters # Note: at this point, there may be a race condition: monitor saved # current woken time but timer.sleep() happen just before monitor # sleep. To fix that, either sleep here or increase time in a loop # to ensure that monitor wakes up at some point. # Try again but already at miniters = 1 so nothing will be done timer.sleep(maxinterval * 2) t.update(2) timeend = timer.time() while not (t.monitor.woken >= timeend): timer.sleep(1) # Force monitor to wake up if it woken too soon sleep(0.000001) # Wait for the monitor to get out of sleep's loop and update tqdm.. assert t.miniters == 1 # check that monitor corrected miniters # Check that class var monitor is deleted if no instance left tqdm.monitor_interval = 10 assert tqdm.monitor is None
def test_thread_map(): """Test contrib.concurrent.thread_map""" with closing(StringIO()) as our_file: a = range(9) b = [i + 1 for i in a] try: assert thread_map(lambda x: x + 1, a, file=our_file) == b except ImportError: raise SkipTest assert thread_map(incr, a, file=our_file) == b
async def test_as_completed(): """Test asyncio as_completed""" with closing(StringIO()) as our_file: t = time() skew = time() - t for i in as_completed([asyncio.sleep(0.01) for _ in range(100)], file=our_file): await i assert time() - t - 2 * skew < (0.01 * 100) / 2, "Assuming >= 2 cores" assert '100/100' in our_file.getvalue()
async def test_coroutines(): """Test asyncio coroutine.send""" with closing(StringIO()) as our_file: with tqdm(count(), file=our_file) as pbar: async for row in pbar: if row == 9: pbar.send(-10) elif row < 0: assert row == -9 break assert '10it' in our_file.getvalue()
async def test_as_completed(): """Test asyncio as_completed""" with closing(StringIO()) as our_file: t = time() skew = time() - t for i in as_completed( [asyncio.sleep(0.01 * i) for i in range(30, 0, -1)], file=our_file): await i assert 0.29 < time() - t - 2 * skew < 0.31 assert '30/30' in our_file.getvalue()
def test_map(): """Test contrib.tmap""" with closing(StringIO()) as our_file: a = range(9) b = [i + 1 for i in a] if sys.version_info[:1] < (3, ): assert tmap(lambda x: x + 1, a, file=our_file) == map(incr, a) else: gen = tmap(lambda x: x + 1, a, file=our_file) assert gen != b assert list(gen) == b
def test_zip(): """Test contrib.tzip""" with closing(StringIO()) as our_file: a = range(9) b = [i + 1 for i in a] if sys.version_info[:1] < (3, ): assert tzip(a, b, file=our_file) == zip(a, b) else: gen = tzip(a, b, file=our_file) assert gen != list(zip(a, b)) assert list(gen) == list(zip(a, b))
async def test_range(): """Test asyncio range""" with closing(StringIO()) as our_file: async for row in tqdm(range(9), desc="range", file=our_file): pass assert '9/9' in our_file.getvalue() our_file.seek(0) our_file.truncate() async for row in trange(9, desc="trange", file=our_file): pass assert '9/9' in our_file.getvalue()
def test_monitoring_multi(): """Test on multiple bars, one not needing miniters adjustment""" # Note: should fix miniters for these tests, else with dynamic_miniters # it's too complicated to handle with monitoring update and maxinterval... maxinterval = 2 total = 1000 # Setup a discrete timer timer = DiscreteTimer() # And a fake sleeper sleeper = FakeSleep(timer) # Setup TMonitor to use the timer TMonitor._time = timer.time TMonitor._event = make_create_fake_sleep_event(sleeper.sleep) # Set monitor interval tqdm.monitor_interval = maxinterval with closing(StringIO()) as our_file: with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, maxinterval=maxinterval) as t1: # Set high maxinterval for t2 so monitor does not need to adjust it with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, maxinterval=1E5) as t2: cpu_timify(t1, timer) cpu_timify(t2, timer) # Do a lot of iterations in a small timeframe timer.sleep(maxinterval / 2) t1.update(500) t2.update(500) assert t1.miniters == 500 assert t2.miniters == 500 # Then do 1 it after monitor interval, so that monitor kicks in timer.sleep(maxinterval * 2) t1.update(1) t2.update(1) # Wait for the monitor to get out of sleep and update tqdm timeend = timer.time() while not (t1.monitor.woken >= timeend and t1.miniters == 1): timer.sleep(1) sleep(0.000001) assert t1.miniters == 1 # check that monitor corrected miniters assert t2.miniters == 500 # check that t2 was not adjusted # Check that class var monitor is deleted if no instance left tqdm.monitor_interval = 10 assert tqdm.monitor is None
def test_pandas_setup(): """Test tqdm.pandas()""" try: from numpy.random import randint import pandas as pd except ImportError: raise SkipTest with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True, total=123) series = pd.Series(randint(0, 50, (100, ))) series.progress_apply(lambda x: x + 10) res = our_file.getvalue() assert '100/123' in res
async def test_generators(): """Test asyncio generators""" with closing(StringIO()) as our_file: async for row in tqdm(count(), desc="counter", file=our_file): if row >= 8: break assert '9it' in our_file.getvalue() our_file.seek(0) our_file.truncate() async for row in tqdm(acount(), desc="async_counter", file=our_file): if row >= 8: break assert '9it' in our_file.getvalue()
def test_pandas_deprecation(): """Test bar object instance as argument deprecation""" try: from numpy.random import randint from tqdm import tqdm_pandas import pandas as pd except ImportError: raise SkipTest with closing(StringIO()) as our_file: tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20)) df = pd.DataFrame(randint(0, 50, (500, 3))) df.groupby(0).progress_apply(lambda x: None) # Check deprecation message assert "TqdmDeprecationWarning" in our_file.getvalue() assert "instead of `tqdm_pandas(tqdm(...))`" in our_file.getvalue() with closing(StringIO()) as our_file: tqdm_pandas(tqdm, file=our_file, leave=False, ascii=True, ncols=20) df = pd.DataFrame(randint(0, 50, (500, 3))) df.groupby(0).progress_apply(lambda x: None) # Check deprecation message assert "TqdmDeprecationWarning" in our_file.getvalue() assert "instead of `tqdm_pandas(tqdm, ...)`" in our_file.getvalue()
def test_pandas_map(): """Test pandas.Series.progress_map""" try: from numpy.random import randint import pandas as pd except: raise SkipTest with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True) dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc')) dfs.a.progress_map(lambda x: None) if our_file.getvalue().count('100%') < 1: raise AssertionError("\nExpected:\n{0}\nIn:{1}\n".format( '100% at least twice', our_file.getvalue()))
def test_pandas_data_frame(): """Test pandas.DataFrame.progress_apply and .progress_applymap""" try: from numpy.random import randint import pandas as pd except ImportError: raise SkipTest with closing(StringIO()) as our_file: tqdm.pandas(file=our_file, leave=True, ascii=True) df = pd.DataFrame(randint(0, 50, (100, 200))) def task_func(x): return x + 1 # applymap res1 = df.progress_applymap(task_func) res2 = df.applymap(task_func) assert res1.equals(res2) # apply unhashable res1 = [] df.progress_apply(res1.extend) assert len(res1) == df.size # apply for axis in [0, 1, 'index', 'columns']: res3 = df.progress_apply(task_func, axis=axis) res4 = df.apply(task_func, axis=axis) assert res3.equals(res4) our_file.seek(0) if our_file.read().count('100%') < 3: our_file.seek(0) raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( '100% at least three times', our_file.read())) # apply_map, apply axis=0, apply axis=1 expects = ['20000/20000', '200/200', '100/100'] for exres in expects: our_file.seek(0) if our_file.getvalue().count(exres) < 1: our_file.seek(0) raise AssertionError( "\nExpected:\n{0}\nIn:\n {1}\n".format( exres + " at least once.", our_file.read()))
def test_pandas_apply_args_deprecation(): """Test warning info in `pandas.Dataframe(Series).progress_apply(func, *args)`""" try: from numpy.random import randint from tqdm import tqdm_pandas import pandas as pd except ImportError: raise SkipTest with closing(StringIO()) as our_file: tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20)) df = pd.DataFrame(randint(0, 50, (500, 3))) df.progress_apply(lambda x: None, 1) # 1 shall cause a warning # Check deprecation message res = our_file.getvalue() assert all([i in res for i in ( "TqdmDeprecationWarning", "not supported", "keyword arguments instead")])
def test_monitoring_and_cleanup(): """Test for stalled tqdm instance and monitor deletion""" # Note: should fix miniters for these tests, else with dynamic_miniters # it's too complicated to handle with monitoring update and maxinterval... maxinterval = tqdm.monitor_interval assert maxinterval == 10 total = 1000 with closing(StringIO()) as our_file: with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, maxinterval=maxinterval) as t: cpu_timify(t, Time) # Do a lot of iterations in a small timeframe # (smaller than monitor interval) Time.fake_sleep(maxinterval / 10) # monitor won't wake up t.update(500) # check that our fixed miniters is still there assert t.miniters <= 500 # TODO: should really be == 500 # Then do 1 it after monitor interval, so that monitor kicks in Time.fake_sleep(maxinterval) t.update(1) # Wait for the monitor to get out of sleep's loop and update tqdm.. timeend = Time.time() while not (t.monitor.woken >= timeend and t.miniters == 1): Time.fake_sleep(1) # Force awake up if it woken too soon assert t.miniters == 1 # check that monitor corrected miniters # Note: at this point, there may be a race condition: monitor saved # current woken time but Time.sleep() happen just before monitor # sleep. To fix that, either sleep here or increase time in a loop # to ensure that monitor wakes up at some point. # Try again but already at miniters = 1 so nothing will be done Time.fake_sleep(maxinterval) t.update(2) timeend = Time.time() while t.monitor.woken < timeend: Time.fake_sleep(1) # Force awake if it woken too soon # Wait for the monitor to get out of sleep's loop and update tqdm assert t.miniters == 1 # check that monitor corrected miniters
def test_pandas_leave(): """Test pandas with `leave=True`""" try: from numpy.random import randint import pandas as pd except ImportError: raise SkipTest with closing(StringIO()) as our_file: df = pd.DataFrame(randint(0, 100, (1000, 6))) tqdm.pandas(file=our_file, leave=True, ascii=True) df.groupby(0).progress_apply(lambda x: None) our_file.seek(0) exres = '100%|##########| 100/100' if exres not in our_file.read(): our_file.seek(0) raise AssertionError("\nExpected:\n{0}\nIn:{1}\n".format( exres, our_file.read()))
def test_pandas_apply(): """ Test pandas.DataFrame.progress_apply """ try: from numpy.random import randint from tqdm import tqdm_pandas import pandas as pd except: raise SkipTest with closing(StringIO()) as our_file: df = pd.DataFrame(randint(0, 100, (1000, 6))) tqdm_pandas(tqdm(file=our_file, leave=True, ascii=True)) df.progress_apply(lambda x: None) our_file.seek(0) if '/6' not in our_file.read(): our_file.seek(0) raise AssertionError("\nExpected:\n{0}\nIn:{1}\n".format( '/6', our_file.read()))
def test_monitoring_multi(): """Test on multiple bars, one not needing miniters adjustment""" # Note: should fix miniters for these tests, else with dynamic_miniters # it's too complicated to handle with monitoring update and maxinterval... maxinterval = tqdm.monitor_interval assert maxinterval == 10 total = 1000 with closing(StringIO()) as our_file: with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, maxinterval=maxinterval) as t1: # Set high maxinterval for t2 so monitor does not need to adjust it with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, maxinterval=1E5) as t2: cpu_timify(t1, Time) cpu_timify(t2, Time) # Do a lot of iterations in a small timeframe Time.fake_sleep(maxinterval / 10) t1.update(500) t2.update(500) assert t1.miniters <= 500 # TODO: should really be == 500 assert t2.miniters == 500 # Then do 1 it after monitor interval, so that monitor kicks in Time.fake_sleep(maxinterval) t1.update(1) t2.update(1) # Wait for the monitor to get out of sleep and update tqdm timeend = Time.time() while not (t1.monitor.woken >= timeend and t1.miniters == 1): Time.fake_sleep(1) assert t1.miniters == 1 # check that monitor corrected miniters assert t2.miniters == 500 # check that t2 was not adjusted