def test_iterator_without_max_value():
'''Progressbar can't guess max_value.'''
p = progressbar.ProgressBar(widgets=[
progressbar.AnimatedMarker(),
progressbar.FormatLabel('%(value)d'),
progressbar.BouncingBar(),
progressbar.BouncingBar(marker=progressbar.RotatingMarker()),
])
for i in p((i for i in range(10))):
time.sleep(0.001)
def rotating_bouncing_marker():
widgets = [progressbar.BouncingBar(marker=progressbar.RotatingMarker())]
with progressbar.ProgressBar(widgets=widgets, max_value=20,
term_width=10) as progress:
for i in range(20):
time.sleep(0.1)
progress.update(i)
widgets = [progressbar.BouncingBar(marker=progressbar.RotatingMarker(),
fill_left=False)]
with progressbar.ProgressBar(widgets=widgets, max_value=20,
term_width=10) as progress:
for i in range(20):
time.sleep(0.1)
progress.update(i)
def _get_progressbar_widgets(
sim_index: Optional[int], timescale: TimeValue,
know_stop_time: bool) -> List[progressbar.widgets.WidgetBase]:
widgets = []
if sim_index is not None:
widgets.append(f'Sim {sim_index:3}|')
magnitude, units = timescale
if magnitude == 1:
sim_time_format = f'%(value)6.0f {units}|'
else:
sim_time_format = f'{magnitude}x%(value)6.0f {units}|'
widgets.append(progressbar.FormatLabel(sim_time_format))
widgets.append(progressbar.Percentage())
if know_stop_time:
widgets.append(progressbar.Bar())
else:
widgets.append(progressbar.BouncingBar())
widgets.append(progressbar.ETA())
return widgets
def extract_metadata(folder_in, file_out):
folder_in = pathlib.Path(folder_in)
file_out = pathlib.Path(file_out)
ensure_path(file_out)
i = 1
widgets = [
'Extracting File # ',
pb.Counter(), ' ',
pb.Timer(), ' ',
pb.BouncingBar(marker='.', left='[', right=']')
]
with pb.ProgressBar(widgets=widgets) as bar:
with file_out.open('w', encoding='utf-8', newline='') as file_out:
writer = csv.writer(file_out,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_ALL)
writer.writerow([
'id', 'referenceCount', 'authorCount', 'figureCount',
'tableCount', 'isFullText', 'title'
])
for file_name in folder_in.iterdir():
if file_name.is_file() and file_name.suffix.lower() == '.json':
bar.update(i)
i = i + 1
article = parse_json_to_article(file_name)
writer.writerow([
article.id, article.references, article.authors,
article.figures, article.tables, article.fullText,
article.title
])
def save_documents(jsonl_out: pathlib.Path,
documents: t.Iterator[dict]) -> None:
"""
Saves the documents to a `JSONL` file
Parameters
----------
jsonl_out : pathlib.Path
The JSONL file to contain all the documents
documents : Iterator[dict]
The JSON documents to save
"""
bar_i = 0
widgets = [
'Saving JSONL # ',
pb.Counter(), ' ',
pb.Timer(), ' ',
pb.BouncingBar(marker='.', left='[', right=']')
]
with pb.ProgressBar(widgets=widgets) as bar:
with open(jsonl_out, 'w', encoding='utf-8') as fp:
with jl.Writer(fp, compact=True, sort_keys=True) as writer:
for document in documents:
writer.write(document)
bar_i = bar_i + 1
bar.update(bar_i)
def unnest_corpus(path_in: pathlib.Path, path_out: pathlib.Path,
control: pathlib.Path) -> None:
u.assert_folder_is_readable(path_in)
u.assert_folder_is_writable(path_out)
i = 1
widgets = [
'Unnesting Corpus # ',
pb.Counter(), ' ',
pb.Timer(), ' ',
pb.BouncingBar(marker='.', left='[', right=']')
]
with pb.ProgressBar(widgets=widgets) as bar:
with control.open('w', encoding='utf-8', newline='') as control:
writer = csv.writer(control,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_ALL)
writer.writerow(['filename', 'relitive path'])
for file_name in path_in.rglob("*"):
if u.is_corpus_document(file_name):
bar.update(i)
i = i + 1
dest_path = path_out.joinpath(
f'{str(uuid.uuid4())}{file_name.suffix}')
shutil.copy(file_name, dest_path)
relative_path = file_name.relative_to(path_in)
writer.writerow([dest_path.name, str(relative_path)])
def gather_files(path, config, file_inventory) -> int:
""":returns file count"""
if not os.path.isdir(path):
logging.error("Not a directory: '%s', skipping indexing", path)
return
logging.info("Indexing %s", path)
logging.info("Calculating number of files to index (.=100files)")
if config.verbose:
widgets = [
" [",
progressbar.Timer(format="Elapsed %(elapsed)s"),
" ",
"count: ",
progressbar.Counter(),
"] ",
progressbar.BouncingBar(),
]
pbar = progressbar.ProgressBar(widgets=widgets)
file_count = 0
for file in NeedsIndexFileGenerator(path, config)():
pickle.dump(file, file_inventory)
file_count += 1
# if config.verbose and (file_count % 100) == 0:
# sys.stdout.write('.')
# sys.stdout.flush()
if config.verbose:
pbar.update(file_count)
# if config.verbose:
# sys.stdout.write('\n')
if config.verbose:
pbar.finish()
file_inventory.seek(0)
return file_count
def test_all_widgets_small_values(max_value):
widgets = [
progressbar.Timer(),
progressbar.ETA(),
progressbar.AdaptiveETA(),
progressbar.AbsoluteETA(),
progressbar.DataSize(),
progressbar.FileTransferSpeed(),
progressbar.AdaptiveTransferSpeed(),
progressbar.AnimatedMarker(),
progressbar.Counter(),
progressbar.Percentage(),
progressbar.FormatLabel('%(value)d'),
progressbar.SimpleProgress(),
progressbar.Bar(),
progressbar.ReverseBar(),
progressbar.BouncingBar(),
progressbar.CurrentTime(),
progressbar.CurrentTime(microseconds=False),
progressbar.CurrentTime(microseconds=True),
]
p = progressbar.ProgressBar(widgets=widgets, max_value=max_value)
for i in range(10):
time.sleep(1)
p.update(i + 1)
p.finish()
def create_bar(self):
"""Create a new progress bar.
Calls `self.get_iter_per_epoch()`, selects an appropriate
set of widgets and creates a ProgressBar.
"""
iter_per_epoch = self.get_iter_per_epoch()
epochs_done = self.main_loop.log.status['epochs_done']
if iter_per_epoch is None:
widgets = [
"Epoch {}, step ".format(epochs_done),
progressbar.Counter(), ' ',
progressbar.BouncingBar(), ' ',
progressbar.Timer()
]
iter_per_epoch = progressbar.UnknownLength
else:
widgets = [
"Epoch {}, step ".format(epochs_done),
progressbar.Counter(), ' (',
progressbar.Percentage(), ') ',
progressbar.Bar(), ' ',
progressbar.Timer(), ' ',
progressbar.ETA()
]
return progressbar.ProgressBar(widgets=widgets,
max_value=iter_per_epoch)
def extract_json_from_jsonl(jsonl_in: pathlib.Path, folder_out: pathlib.Path,
id_element: str) -> None:
"""
Extracts a folder of `JSON` files from a a `JSONL` file.
Parameters
----------
jsonl_in : pathlib.Path
The JSONL containing all the documents
folder_out : pathlib.Path
The folder containing all the documents after being extracted
id_element : str
The name of the element to use as a file name
"""
folder_out.mkdir(parents=True, exist_ok=True)
bar_i = 0
widgets = [
'Saving JSONL # ',
pb.Counter(), ' ',
pb.Timer(), ' ',
pb.BouncingBar(marker='.', left='[', right=']')
]
with pb.ProgressBar(widgets=widgets) as bar:
documents = u.list_jsonl_documents(jsonl_in)
for document in documents:
if id_element in document:
file_name = folder_out.joinpath(
f'./{document[id_element]}.json')
with open(file_name, 'w', encoding='utf-8') as fp:
json.dump(document, fp, sort_keys=True, indent=None)
bar_i = bar_i + 1
bar.update(bar_i)
def dispatch_ip_tick_hook(
self,
ipaddr,
node, # pylint: disable=w0613
message,
timestamp, # pylint: disable=w0613
duration): # pylint: disable=w0613
# start progressbar
if self.pbar is None:
widgets = [
'Collecting image : ',
# progressbar.BouncingBar(marker=progressbar.RotatingMarker()),
progressbar.BouncingBar(marker='*'),
progressbar.FormatLabel(' %(seconds).2fs'),
]
self.pbar = \
progressbar.ProgressBar(widgets=widgets,
maxval=progressbar.UnknownLength)
self.pbar.start()
# progressbar is willing to work as expected here
# with maxval=UnknownLength
# but still insists to get a real value apparently
self.value = 0
self.value += 1
self.pbar.update(self.value)
# hack way to finish the progressbar
# since we have no other way to figure it out
if message['tick'] == 'END':
self.pbar.finish()
def save_documents(jsont_out: pathlib.Path,
documents: t.Iterator[dict]) -> None:
"""
Saves the documents to a `JSONT` file
Parameters
----------
jsonl_out : pathlib.Path
The JSONL file to contain all the documents
documents : Iterator[dict]
The JSON documents to save
"""
bar_i = 0
widgets = [
'Saving JSONT # ',
pb.Counter(), ' ',
pb.Timer(), ' ',
pb.BouncingBar(marker='.', left='[', right=']')
]
with pb.ProgressBar(widgets=widgets) as bar:
with tf.open(jsont_out, 'w') as tar_ball:
for document in documents:
txt = json.dumps(document, indent=None, sort_keys=True)
txt = txt.encode('utf-8')
with BytesIO(txt) as tar_file:
info = tf.TarInfo(name=f'{bar_i}.json')
info.size = len(txt)
tar_ball.addfile(info, fileobj=tar_file)
bar_i = bar_i + 1
bar.update(bar_i)
def test_all_widgets_max_width(max_width, term_width):
widgets = [
progressbar.Timer(max_width=max_width),
progressbar.ETA(max_width=max_width),
progressbar.AdaptiveETA(max_width=max_width),
progressbar.AbsoluteETA(max_width=max_width),
progressbar.DataSize(max_width=max_width),
progressbar.FileTransferSpeed(max_width=max_width),
progressbar.AdaptiveTransferSpeed(max_width=max_width),
progressbar.AnimatedMarker(max_width=max_width),
progressbar.Counter(max_width=max_width),
progressbar.Percentage(max_width=max_width),
progressbar.FormatLabel('%(value)d', max_width=max_width),
progressbar.SimpleProgress(max_width=max_width),
progressbar.Bar(max_width=max_width),
progressbar.ReverseBar(max_width=max_width),
progressbar.BouncingBar(max_width=max_width),
progressbar.FormatCustomText('Custom %(text)s',
dict(text='text'),
max_width=max_width),
progressbar.DynamicMessage('custom', max_width=max_width),
progressbar.CurrentTime(max_width=max_width),
]
p = progressbar.ProgressBar(widgets=widgets, term_width=term_width)
p.update(0)
p.update()
for widget in p._format_widgets():
if max_width and max_width < term_width:
assert widget == ''
else:
assert widget != ''
def test_all_widgets_large_values(max_value):
widgets = [
progressbar.Timer(),
progressbar.ETA(),
progressbar.AdaptiveETA(),
progressbar.AbsoluteETA(),
progressbar.DataSize(),
progressbar.FileTransferSpeed(),
progressbar.AdaptiveTransferSpeed(),
progressbar.AnimatedMarker(),
progressbar.Counter(),
progressbar.Percentage(),
progressbar.FormatLabel('%(value)d/%(max_value)d'),
progressbar.SimpleProgress(),
progressbar.Bar(fill=lambda progress, data, width: '#'),
progressbar.ReverseBar(),
progressbar.BouncingBar(),
progressbar.FormatCustomText('Custom %(text)s', dict(text='text')),
]
p = progressbar.ProgressBar(widgets=widgets, max_value=max_value)
p.update()
time.sleep(1)
p.update()
for i in range(0, 10**6, 10**4):
time.sleep(1)
p.update(i)
def processReplay(self, infile: Path):
# FIXME: Sinks need to support progress bar.
widgets = [
progressbar.FormatLabel(
f'Converting to {self.args.format.upper()}'),
progressbar.BouncingBar(),
progressbar.FormatLabel(' Elapsed: %(elapsed)s'),
]
with progressbar.ProgressBar(widgets=widgets) as progress:
print(f"[*] Converting '{infile}' to {self.args.format.upper()}")
outfile = self.prefix + infile.stem
sink, outfile = getSink(self.args.format,
outfile,
progress=lambda: progress.update(0))
if not sink:
print(
"The input file is already a replay file. Nothing to do.")
sys.exit(1)
fd = open(infile, "rb")
replay = Replay(fd, handler=sink)
print(f"\n[+] Succesfully wrote '{outfile}'")
sink.cleanup()
fd.close()
def nest_corpus(path_in: pathlib.Path, path_out: pathlib.Path,
control: pathlib.Path) -> None:
u.assert_folder_is_readable(path_in)
u.assert_folder_is_writable(path_out)
i = 1
widgets = [
'Re-nesting Corpus # ',
pb.Counter(), ' ',
pb.Timer(), ' ',
pb.BouncingBar(marker='.', left='[', right=']')
]
with pb.ProgressBar(widgets=widgets) as bar:
with control.open('r', encoding='utf-8', newline='') as control:
reader = csv.DictReader(control,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_ALL)
for row in reader:
bar.update(i)
i = i + 1
source_path = path_in.joinpath(row['filename'])
dest_path = path_out.joinpath(row['relitive path'])
pathlib.Path(dest_path.parent).mkdir(parents=True,
exist_ok=True)
shutil.copy(source_path, dest_path)
def ProgressIndicator(label, maxval=1000000):
widgets = [
pb.FormatLabel(label),
pb.BouncingBar(marker=pb.RotatingMarker())
]
pind = pb.ProgressBar(widgets=widgets, maxval=maxval).start()
return pind
def format_label_rotating_bouncer():
widgets = [progressbar.FormatLabel('Animated Bouncer: value %(value)d - '),
progressbar.BouncingBar(marker=progressbar.RotatingMarker())]
bar = progressbar.ProgressBar(widgets=widgets)
for i in bar((i for i in range(18))):
time.sleep(0.1)
def format_label_bouncer():
widgets = [
progressbar.FormatLabel('Bouncer: value %(value)d - '),
progressbar.BouncingBar(),
]
bar = progressbar.ProgressBar(widgets=widgets)
for i in bar((i for i in range(100))):
time.sleep(0.01)
def test_left_justify():
'''Left justify using the terminal width'''
p = progressbar.ProgressBar(
widgets=[progressbar.BouncingBar(marker=progressbar.RotatingMarker())],
max_value=100, term_width=20, left_justify=True)
assert p.term_width is not None
for i in range(100):
p.update(i)
def test_fill_left():
'''Right justify using the terminal width'''
p = progressbar.ProgressBar(
widgets=[progressbar.BouncingBar(fill_left=True)],
max_value=100, term_width=20)
assert p.term_width is not None
for i in range(100):
p.update(i)
def do_mmseqs_db(mmseqdb, prt_path, logmmseq, quiet):
"""
Runs create_mmseqs_db with an "infinite progress bar" in the background.
create_mmseqs_db does :
Create ffindex of protein bank (prt_path) if not already done. If done, just write a message
to tell the user that the current existing file will be used.
Parameters
----------
mmseqdb : str
path to base filename for output of mmseqs createdb
prt_path : str
path to the file containing all proteins to cluster
logmmseq : str
path to file where logs must be written
quiet : bool
True if no output in stderr/stdout, False otherwise
Returns
-------
bool
True if mmseqs db just created, False if already existed
"""
logger.info("Creating database")
try:
stop_bar = False
if quiet:
widgets = []
# If not quiet, start a progress bar while clustering proteins. We cannot guess
# how many time it will take, so we start an "infinite" bar, and send it a signal
# when it has to stop. If quiet, we start a thread that will immediatly stop
else:
widgets = [
progressbar.BouncingBar(marker=progressbar.RotatingMarker(
markers="◐◓◑◒")), " - ",
progressbar.Timer()
]
x = threading.Thread(target=utils.thread_progressbar,
args=(
widgets,
lambda: stop_bar,
))
x.start()
res = create_mmseqs_db(mmseqdb, prt_path, logmmseq)
# except KeyboardInterrupt: # pragma: no cover
except: # pragma: no cover
stop_bar = True
x.join()
sys.exit(1)
# Clustering done, stop bar and join (if quiet, it was already finished, so we just join it)
stop_bar = True
x.join()
return res
def documents_to_corpus(path_in: pathlib.Path, path_out: pathlib.Path) -> None:
u.assert_folder_is_readable(path_in)
u.assert_folder_is_writable(path_out)
i = 1
widgets = [ 'Formatting Document # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker = '.', left = '[', right = ']')]
with pb.ProgressBar(widgets = widgets) as bar:
for file_name in path_in.iterdir():
if u.is_corpus_document(file_name):
bar.update(i)
i = i + 1
sentences = __tokenize_document(file_name)
u.write_document(path_out, file_name, sentences)
def progress_overlay(items: t.Iterator, title: str) -> t.Iterator:
bar_i = 0
widgets = [
title, ' ',
pb.Counter(), ' ',
pb.Timer(), ' ',
pb.BouncingBar(marker='.', left='[', right=']')
]
with pb.ProgressBar(widgets=widgets) as bar:
for item in items:
bar_i = bar_i + 1
bar.update(bar_i)
yield item
def test_no_fill(monkeypatch):
'''Simply bounce within the terminal width'''
bar = progressbar.BouncingBar()
bar.INTERVAL = timedelta(seconds=1)
p = progressbar.ProgressBar(widgets=[bar],
max_value=progressbar.UnknownLength,
term_width=20)
assert p.term_width is not None
for i in range(30):
p.update(i, force=True)
# Fake the start time so we can actually emulate a moving progress bar
p.start_time = p.start_time - timedelta(seconds=i)
def _progress(items: t.Iterator) -> t.Iterator:
bar_i = 0
widgets = [
'Saving Items # ',
pb.Counter(), ' ',
pb.Timer(), ' ',
pb.BouncingBar(marker='.', left='[', right=']')
]
with pb.ProgressBar(widgets=widgets) as bar:
for item in items:
bar_i = bar_i + 1
bar.update(bar_i)
yield item
def vectorize_corpus(path_in: pathlib.Path, path_out: pathlib.Path, path_control: pathlib.Path) -> None:
u.assert_folder_is_readable(path_in)
u.assert_folder_is_writable(path_out)
i = 1
widgets = [ 'Collecting Tokens # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker = '.', left = '[', right = ']')]
tokens = dict()
with pb.ProgressBar(widgets = widgets) as bar:
for file_name in path_in.iterdir():
if u.is_corpus_document(file_name):
bar.update(i)
i = i + 1
__append_tokens(tokens, file_name)
token_map = __map_tokens(tokens)
__save_token_file(tokens, token_map, path_control)
i = 1
widgets = [ 'Vectorizing Document # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker = '.', left = '[', right = ']')]
with pb.ProgressBar(widgets = widgets) as bar:
for file_name in path_in.iterdir():
if u.is_corpus_document(file_name):
bar.update(i)
i = i + 1
vector = __vectorise_document(file_name, token_map)
u.write_document(path_out, file_name, vector)
def example23():
"""
Display progress bar using tqdm.
>>> example23()
True
"""
widgets = [progressbar.BouncingBar(marker=progressbar.RotatingMarker())]
with progressbar.ProgressBar(widgets=widgets, max_value=20,
term_width=10) as progress:
for i in range(20):
sleep(0.1)
progress.update(i)
widgets = [
progressbar.BouncingBar(marker=progressbar.RotatingMarker(),
fill_left=False)
]
with progressbar.ProgressBar(widgets=widgets, max_value=20,
term_width=10) as progress:
for i in range(20):
sleep(0.1)
progress.update(i)
return True
def example16():
"""
Display progress bar using tqdm.
>>> example16()
True
"""
widgets = [
progressbar.FormatLabel("Bouncer: value %(value)d - "),
progressbar.BouncingBar()
]
pbar = progressbar.ProgressBar(widgets=widgets)
for i in pbar((i for i in range(25))):
sleep(0.2)
return True
def upload_to_solr(filename):
"""Commit a CSV file to Solr using a stream"""
prog_widgets = [
'Uploading... ',
progressbar.BouncingBar(), ' ',
progressbar.Timer(format='Time: %s')
]
prog_bar = progressbar.ProgressBar(widgets=prog_widgets)
prog_bar.start()
# Build the Solr upload URL
url = (
'"{server}/update?stream.file={path}&stream.contentType=text/csv;charset=utf-8&commit=true"'
.format(server=settings.SOLR_SERVER, path=os.path.abspath(filename)))
command = 'curl -s -o /dev/null -w "%{http_code}" ' + url
proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
polls = 0
while proc.returncode is None:
proc.poll()
polls += 1
prog_bar.update((polls * UPLOAD_PROGRESS_STEP) % prog_bar.maxval)
time.sleep(UPLOAD_POLL_WAIT_SECS)
prog_bar.finish()
if proc.returncode != 0:
failure_message = 'process returned {}'.format(proc.returncode)
else:
status = proc.communicate()[0]
if status[0] != '2':
failure_message = 'status {}'.format(status)
else:
failure_message = None
if failure_message is not None:
print 'Upload failed ({}). See the Solr logs for details.'.format(
failure_message)
else:
print 'Upload successful.'
