def __init__(
self,
*,
upd=None,
fstags=None,
all_progress=None,
ydl_opts=None,
parallel=DEFAULT_PARALLEL,
):
if upd is None:
upd = Upd()
if all_progress is None:
all_progress = OverProgress()
self.upd = upd
self.sem = Semaphore(parallel)
self.proxy0 = upd.insert(0)
self.fstags = fstags
self.all_progress = all_progress
self.ydl_opts = ydl_opts
self.Rs = []
self.nfetches = 0
self._lock = RLock()
@logexc
def update0():
nfetches = self.nfetches
if nfetches == 0:
self.proxy0("Idle.")
else:
self.proxy0(
self.all_progress.status(
"%d %s" % (nfetches, "fetch" if nfetches == 1 else "fetches"),
upd.columns - 1
)
)
self.update0 = update0
update0()
def iterbar(
self,
it,
label=None,
upd=None,
proxy=None,
itemlenfunc=None,
statusfunc=None,
incfirst=False,
width=None,
window=None,
update_frequency=1,
update_min_size=None,
report_print=None,
runstate=None,
):
''' An iterable progress bar: a generator yielding values
from the iterable `it` while updating a progress bar.
Parameters:
* `it`: the iterable to consume and yield.
* `itemlenfunc`: an optional function returning the "size" of each item
from `it`, used to advance `self.position`.
The default is to assume a size of `1`.
A convenient alternative choice may be the builtin function `len`.
* `incfirst`: whether to advance `self.position` before we
`yield` an item from `it` or afterwards.
This reflects whether it is considered that progress is
made as items are obtained or only after items are processed
by whatever is consuming this generator.
The default is `False`, advancing after processing.
* `label`: a label for the progress bar,
default from `self.name`.
* `width`: an optional width expressing how wide the progress bar
text may be.
The default comes from the `proxy.width` property.
* `window`: optional timeframe to define "recent" in seconds;
if the default `statusfunc` (`Progress.status`) is used
this is passed to it
* `statusfunc`: an optional function to compute the progress bar text
accepting `(self,label,width)`.
* `proxy`: an optional proxy for displaying the progress bar,
a callable accepting the result of `statusfunc`.
The default is a `cs.upd.UpdProxy` created from `upd`,
which inserts a progress bar above the main status line.
* `upd`: an optional `cs.upd.Upd` instance,
used only to produce the default `proxy` if that is not supplied.
The default `upd` is `cs.upd.Upd()`
which uses `sys.stderr` for display.
* `update_frequency`: optional update frequency, default `1`;
only update the progress bar after this many iterations,
useful if the iteration rate is quite high
* `update_min_size`: optional update step size;
only update the progress bar after an advance of this many units,
useful if the iteration size increment is quite small
* `report_print`: optional `print` compatible function
with which to write a report on completion;
this may also be a `bool`, which if true will use `Upd.print`
in order to interoperate with `Upd`.
* `runstate`: optional `RunState` whose `.cancelled` property can be consulted
Example use:
from cs.units import DECIMAL_SCALE
rows = [some list of data]
P = Progress(total=len(rows), units_scale=DECIMAL_SCALE)
for row in P.iterbar(rows, incfirst=True):
... do something with each row ...
f = open(data_filename, 'rb')
datalen = os.stat(f).st_size
def readfrom(f):
while True:
bs = f.read(65536)
if not bs:
break
yield bs
P = Progress(total=datalen)
for bs in P.iterbar(readfrom(f), itemlenfunc=len):
... process the file data in bs ...
'''
if label is None:
label = self.name
delete_proxy = False
if proxy is None:
if upd is None:
upd = Upd()
proxy = upd.insert(1)
delete_proxy = True
if statusfunc is None:
statusfunc = lambda P, label, width: P.status(
label, width, window=window)
iteration = 0
last_update_iteration = 0
last_update_pos = start_pos = self.position
def update_status(force=False):
nonlocal self, proxy, statusfunc, label, width
nonlocal iteration, last_update_iteration, last_update_pos
if (force or iteration - last_update_iteration >= update_frequency
or
(update_min_size is not None
and self.position - last_update_pos >= update_min_size)):
last_update_iteration = iteration
last_update_pos = self.position
proxy(statusfunc(self, label, width or proxy.width))
update_status(True)
for iteration, item in enumerate(it):
length = itemlenfunc(item) if itemlenfunc else 1
if incfirst:
self += length
update_status()
yield item
if not incfirst:
self += length
update_status()
if runstate is not None and runstate.cancelled:
break
if delete_proxy:
proxy.delete()
else:
update_status(True)
if report_print:
if isinstance(report_print, bool):
report_print = print
report_print(
label + (': (cancelled)' if runstate is not None
and runstate.cancelled else ':'),
self.format_counter(self.position - start_pos), 'in',
transcribe(self.elapsed_time,
TIME_SCALE,
max_parts=2,
skip_zero=True))
def bar(
self,
label=None,
*,
upd=None,
proxy=None,
statusfunc=None,
width=None,
window=None,
report_print=None,
insert_pos=1,
deferred=False,
):
''' A context manager to create and withdraw a progress bar.
It returns the `UpdProxy` which displays the progress bar.
Parameters:
* `label`: a label for the progress bar,
default from `self.name`.
* `proxy`: an optional `UpdProxy` to display the progress bar
* `upd`: an optional `cs.upd.Upd` instance,
used to produce the progress bar status line if not supplied.
The default `upd` is `cs.upd.Upd()`
which uses `sys.stderr` for display.
* `statusfunc`: an optional function to compute the progress bar text
accepting `(self,label,width)`.
* `width`: an optional width expressing how wide the progress bar
text may be.
The default comes from the `proxy.width` property.
* `window`: optional timeframe to define "recent" in seconds;
if the default `statusfunc` (`Progress.status`) is used
this is passed to it
* `report_print`: optional `print` compatible function
with which to write a report on completion;
this may also be a `bool`, which if true will use `Upd.print`
in order to interoperate with `Upd`.
* `insert_pos`: where to insert the progress bar, default `1`
* `deferred`: optional flag; if true do not create the
progress bar until the first update occurs.
Example use:
# display progress reporting during upload_filename()
# which updates the supplied Progress instance
# during its operation
P = Progress(name=label)
with P.bar(report_print=True):
upload_filename(src, progress=P)
'''
if label is None:
label = self.name
if upd is None:
upd = Upd()
if statusfunc is None:
statusfunc = lambda P, label, width: P.status(
label, width, window=window)
pproxy = [proxy]
proxy_delete = proxy is None
def update(P, _):
proxy = pproxy[0]
if proxy is None:
proxy = pproxy[0] = upd.insert(insert_pos, 'LABEL=' + label)
proxy(statusfunc(P, label, width or proxy.width))
try:
if not deferred:
if proxy is None:
proxy = pproxy[0] = upd.insert(insert_pos)
status = statusfunc(self, label, width or proxy.width)
proxy(status)
self.notify_update.add(update)
start_pos = self.position
yield pproxy[0]
finally:
self.notify_update.remove(update)
if proxy and proxy_delete:
proxy.delete()
if report_print:
if isinstance(report_print, bool):
report_print = print
report_print(
label + ':', self.format_counter(self.position - start_pos),
'in',
transcribe(self.elapsed_time,
TIME_SCALE,
max_parts=2,
skip_zero=True))