def test_iterator_without_max_value(): '''Progressbar can't guess max_value.''' p = progressbar.ProgressBar(widgets=[ progressbar.AnimatedMarker(), progressbar.FormatLabel('%(value)d'), progressbar.BouncingBar(), progressbar.BouncingBar(marker=progressbar.RotatingMarker()), ]) for i in p((i for i in range(10))): time.sleep(0.001)
def rotating_bouncing_marker(): widgets = [progressbar.BouncingBar(marker=progressbar.RotatingMarker())] with progressbar.ProgressBar(widgets=widgets, max_value=20, term_width=10) as progress: for i in range(20): time.sleep(0.1) progress.update(i) widgets = [progressbar.BouncingBar(marker=progressbar.RotatingMarker(), fill_left=False)] with progressbar.ProgressBar(widgets=widgets, max_value=20, term_width=10) as progress: for i in range(20): time.sleep(0.1) progress.update(i)
def _get_progressbar_widgets( sim_index: Optional[int], timescale: TimeValue, know_stop_time: bool) -> List[progressbar.widgets.WidgetBase]: widgets = [] if sim_index is not None: widgets.append(f'Sim {sim_index:3}|') magnitude, units = timescale if magnitude == 1: sim_time_format = f'%(value)6.0f {units}|' else: sim_time_format = f'{magnitude}x%(value)6.0f {units}|' widgets.append(progressbar.FormatLabel(sim_time_format)) widgets.append(progressbar.Percentage()) if know_stop_time: widgets.append(progressbar.Bar()) else: widgets.append(progressbar.BouncingBar()) widgets.append(progressbar.ETA()) return widgets
def extract_metadata(folder_in, file_out): folder_in = pathlib.Path(folder_in) file_out = pathlib.Path(file_out) ensure_path(file_out) i = 1 widgets = [ 'Extracting File # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker='.', left='[', right=']') ] with pb.ProgressBar(widgets=widgets) as bar: with file_out.open('w', encoding='utf-8', newline='') as file_out: writer = csv.writer(file_out, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) writer.writerow([ 'id', 'referenceCount', 'authorCount', 'figureCount', 'tableCount', 'isFullText', 'title' ]) for file_name in folder_in.iterdir(): if file_name.is_file() and file_name.suffix.lower() == '.json': bar.update(i) i = i + 1 article = parse_json_to_article(file_name) writer.writerow([ article.id, article.references, article.authors, article.figures, article.tables, article.fullText, article.title ])
def save_documents(jsonl_out: pathlib.Path, documents: t.Iterator[dict]) -> None: """ Saves the documents to a `JSONL` file Parameters ---------- jsonl_out : pathlib.Path The JSONL file to contain all the documents documents : Iterator[dict] The JSON documents to save """ bar_i = 0 widgets = [ 'Saving JSONL # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker='.', left='[', right=']') ] with pb.ProgressBar(widgets=widgets) as bar: with open(jsonl_out, 'w', encoding='utf-8') as fp: with jl.Writer(fp, compact=True, sort_keys=True) as writer: for document in documents: writer.write(document) bar_i = bar_i + 1 bar.update(bar_i)
def unnest_corpus(path_in: pathlib.Path, path_out: pathlib.Path, control: pathlib.Path) -> None: u.assert_folder_is_readable(path_in) u.assert_folder_is_writable(path_out) i = 1 widgets = [ 'Unnesting Corpus # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker='.', left='[', right=']') ] with pb.ProgressBar(widgets=widgets) as bar: with control.open('w', encoding='utf-8', newline='') as control: writer = csv.writer(control, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) writer.writerow(['filename', 'relitive path']) for file_name in path_in.rglob("*"): if u.is_corpus_document(file_name): bar.update(i) i = i + 1 dest_path = path_out.joinpath( f'{str(uuid.uuid4())}{file_name.suffix}') shutil.copy(file_name, dest_path) relative_path = file_name.relative_to(path_in) writer.writerow([dest_path.name, str(relative_path)])
def gather_files(path, config, file_inventory) -> int: """:returns file count""" if not os.path.isdir(path): logging.error("Not a directory: '%s', skipping indexing", path) return logging.info("Indexing %s", path) logging.info("Calculating number of files to index (.=100files)") if config.verbose: widgets = [ " [", progressbar.Timer(format="Elapsed %(elapsed)s"), " ", "count: ", progressbar.Counter(), "] ", progressbar.BouncingBar(), ] pbar = progressbar.ProgressBar(widgets=widgets) file_count = 0 for file in NeedsIndexFileGenerator(path, config)(): pickle.dump(file, file_inventory) file_count += 1 # if config.verbose and (file_count % 100) == 0: # sys.stdout.write('.') # sys.stdout.flush() if config.verbose: pbar.update(file_count) # if config.verbose: # sys.stdout.write('\n') if config.verbose: pbar.finish() file_inventory.seek(0) return file_count
def test_all_widgets_small_values(max_value): widgets = [ progressbar.Timer(), progressbar.ETA(), progressbar.AdaptiveETA(), progressbar.AbsoluteETA(), progressbar.DataSize(), progressbar.FileTransferSpeed(), progressbar.AdaptiveTransferSpeed(), progressbar.AnimatedMarker(), progressbar.Counter(), progressbar.Percentage(), progressbar.FormatLabel('%(value)d'), progressbar.SimpleProgress(), progressbar.Bar(), progressbar.ReverseBar(), progressbar.BouncingBar(), progressbar.CurrentTime(), progressbar.CurrentTime(microseconds=False), progressbar.CurrentTime(microseconds=True), ] p = progressbar.ProgressBar(widgets=widgets, max_value=max_value) for i in range(10): time.sleep(1) p.update(i + 1) p.finish()
def create_bar(self): """Create a new progress bar. Calls `self.get_iter_per_epoch()`, selects an appropriate set of widgets and creates a ProgressBar. """ iter_per_epoch = self.get_iter_per_epoch() epochs_done = self.main_loop.log.status['epochs_done'] if iter_per_epoch is None: widgets = [ "Epoch {}, step ".format(epochs_done), progressbar.Counter(), ' ', progressbar.BouncingBar(), ' ', progressbar.Timer() ] iter_per_epoch = progressbar.UnknownLength else: widgets = [ "Epoch {}, step ".format(epochs_done), progressbar.Counter(), ' (', progressbar.Percentage(), ') ', progressbar.Bar(), ' ', progressbar.Timer(), ' ', progressbar.ETA() ] return progressbar.ProgressBar(widgets=widgets, max_value=iter_per_epoch)
def extract_json_from_jsonl(jsonl_in: pathlib.Path, folder_out: pathlib.Path, id_element: str) -> None: """ Extracts a folder of `JSON` files from a a `JSONL` file. Parameters ---------- jsonl_in : pathlib.Path The JSONL containing all the documents folder_out : pathlib.Path The folder containing all the documents after being extracted id_element : str The name of the element to use as a file name """ folder_out.mkdir(parents=True, exist_ok=True) bar_i = 0 widgets = [ 'Saving JSONL # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker='.', left='[', right=']') ] with pb.ProgressBar(widgets=widgets) as bar: documents = u.list_jsonl_documents(jsonl_in) for document in documents: if id_element in document: file_name = folder_out.joinpath( f'./{document[id_element]}.json') with open(file_name, 'w', encoding='utf-8') as fp: json.dump(document, fp, sort_keys=True, indent=None) bar_i = bar_i + 1 bar.update(bar_i)
def dispatch_ip_tick_hook( self, ipaddr, node, # pylint: disable=w0613 message, timestamp, # pylint: disable=w0613 duration): # pylint: disable=w0613 # start progressbar if self.pbar is None: widgets = [ 'Collecting image : ', # progressbar.BouncingBar(marker=progressbar.RotatingMarker()), progressbar.BouncingBar(marker='*'), progressbar.FormatLabel(' %(seconds).2fs'), ] self.pbar = \ progressbar.ProgressBar(widgets=widgets, maxval=progressbar.UnknownLength) self.pbar.start() # progressbar is willing to work as expected here # with maxval=UnknownLength # but still insists to get a real value apparently self.value = 0 self.value += 1 self.pbar.update(self.value) # hack way to finish the progressbar # since we have no other way to figure it out if message['tick'] == 'END': self.pbar.finish()
def save_documents(jsont_out: pathlib.Path, documents: t.Iterator[dict]) -> None: """ Saves the documents to a `JSONT` file Parameters ---------- jsonl_out : pathlib.Path The JSONL file to contain all the documents documents : Iterator[dict] The JSON documents to save """ bar_i = 0 widgets = [ 'Saving JSONT # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker='.', left='[', right=']') ] with pb.ProgressBar(widgets=widgets) as bar: with tf.open(jsont_out, 'w') as tar_ball: for document in documents: txt = json.dumps(document, indent=None, sort_keys=True) txt = txt.encode('utf-8') with BytesIO(txt) as tar_file: info = tf.TarInfo(name=f'{bar_i}.json') info.size = len(txt) tar_ball.addfile(info, fileobj=tar_file) bar_i = bar_i + 1 bar.update(bar_i)
def test_all_widgets_max_width(max_width, term_width): widgets = [ progressbar.Timer(max_width=max_width), progressbar.ETA(max_width=max_width), progressbar.AdaptiveETA(max_width=max_width), progressbar.AbsoluteETA(max_width=max_width), progressbar.DataSize(max_width=max_width), progressbar.FileTransferSpeed(max_width=max_width), progressbar.AdaptiveTransferSpeed(max_width=max_width), progressbar.AnimatedMarker(max_width=max_width), progressbar.Counter(max_width=max_width), progressbar.Percentage(max_width=max_width), progressbar.FormatLabel('%(value)d', max_width=max_width), progressbar.SimpleProgress(max_width=max_width), progressbar.Bar(max_width=max_width), progressbar.ReverseBar(max_width=max_width), progressbar.BouncingBar(max_width=max_width), progressbar.FormatCustomText('Custom %(text)s', dict(text='text'), max_width=max_width), progressbar.DynamicMessage('custom', max_width=max_width), progressbar.CurrentTime(max_width=max_width), ] p = progressbar.ProgressBar(widgets=widgets, term_width=term_width) p.update(0) p.update() for widget in p._format_widgets(): if max_width and max_width < term_width: assert widget == '' else: assert widget != ''
def test_all_widgets_large_values(max_value): widgets = [ progressbar.Timer(), progressbar.ETA(), progressbar.AdaptiveETA(), progressbar.AbsoluteETA(), progressbar.DataSize(), progressbar.FileTransferSpeed(), progressbar.AdaptiveTransferSpeed(), progressbar.AnimatedMarker(), progressbar.Counter(), progressbar.Percentage(), progressbar.FormatLabel('%(value)d/%(max_value)d'), progressbar.SimpleProgress(), progressbar.Bar(fill=lambda progress, data, width: '#'), progressbar.ReverseBar(), progressbar.BouncingBar(), progressbar.FormatCustomText('Custom %(text)s', dict(text='text')), ] p = progressbar.ProgressBar(widgets=widgets, max_value=max_value) p.update() time.sleep(1) p.update() for i in range(0, 10**6, 10**4): time.sleep(1) p.update(i)
def processReplay(self, infile: Path): # FIXME: Sinks need to support progress bar. widgets = [ progressbar.FormatLabel( f'Converting to {self.args.format.upper()}'), progressbar.BouncingBar(), progressbar.FormatLabel(' Elapsed: %(elapsed)s'), ] with progressbar.ProgressBar(widgets=widgets) as progress: print(f"[*] Converting '{infile}' to {self.args.format.upper()}") outfile = self.prefix + infile.stem sink, outfile = getSink(self.args.format, outfile, progress=lambda: progress.update(0)) if not sink: print( "The input file is already a replay file. Nothing to do.") sys.exit(1) fd = open(infile, "rb") replay = Replay(fd, handler=sink) print(f"\n[+] Succesfully wrote '{outfile}'") sink.cleanup() fd.close()
def nest_corpus(path_in: pathlib.Path, path_out: pathlib.Path, control: pathlib.Path) -> None: u.assert_folder_is_readable(path_in) u.assert_folder_is_writable(path_out) i = 1 widgets = [ 'Re-nesting Corpus # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker='.', left='[', right=']') ] with pb.ProgressBar(widgets=widgets) as bar: with control.open('r', encoding='utf-8', newline='') as control: reader = csv.DictReader(control, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL) for row in reader: bar.update(i) i = i + 1 source_path = path_in.joinpath(row['filename']) dest_path = path_out.joinpath(row['relitive path']) pathlib.Path(dest_path.parent).mkdir(parents=True, exist_ok=True) shutil.copy(source_path, dest_path)
def ProgressIndicator(label, maxval=1000000): widgets = [ pb.FormatLabel(label), pb.BouncingBar(marker=pb.RotatingMarker()) ] pind = pb.ProgressBar(widgets=widgets, maxval=maxval).start() return pind
def format_label_rotating_bouncer(): widgets = [progressbar.FormatLabel('Animated Bouncer: value %(value)d - '), progressbar.BouncingBar(marker=progressbar.RotatingMarker())] bar = progressbar.ProgressBar(widgets=widgets) for i in bar((i for i in range(18))): time.sleep(0.1)
def format_label_bouncer(): widgets = [ progressbar.FormatLabel('Bouncer: value %(value)d - '), progressbar.BouncingBar(), ] bar = progressbar.ProgressBar(widgets=widgets) for i in bar((i for i in range(100))): time.sleep(0.01)
def test_left_justify(): '''Left justify using the terminal width''' p = progressbar.ProgressBar( widgets=[progressbar.BouncingBar(marker=progressbar.RotatingMarker())], max_value=100, term_width=20, left_justify=True) assert p.term_width is not None for i in range(100): p.update(i)
def test_fill_left(): '''Right justify using the terminal width''' p = progressbar.ProgressBar( widgets=[progressbar.BouncingBar(fill_left=True)], max_value=100, term_width=20) assert p.term_width is not None for i in range(100): p.update(i)
def do_mmseqs_db(mmseqdb, prt_path, logmmseq, quiet): """ Runs create_mmseqs_db with an "infinite progress bar" in the background. create_mmseqs_db does : Create ffindex of protein bank (prt_path) if not already done. If done, just write a message to tell the user that the current existing file will be used. Parameters ---------- mmseqdb : str path to base filename for output of mmseqs createdb prt_path : str path to the file containing all proteins to cluster logmmseq : str path to file where logs must be written quiet : bool True if no output in stderr/stdout, False otherwise Returns ------- bool True if mmseqs db just created, False if already existed """ logger.info("Creating database") try: stop_bar = False if quiet: widgets = [] # If not quiet, start a progress bar while clustering proteins. We cannot guess # how many time it will take, so we start an "infinite" bar, and send it a signal # when it has to stop. If quiet, we start a thread that will immediatly stop else: widgets = [ progressbar.BouncingBar(marker=progressbar.RotatingMarker( markers="◐◓◑◒")), " - ", progressbar.Timer() ] x = threading.Thread(target=utils.thread_progressbar, args=( widgets, lambda: stop_bar, )) x.start() res = create_mmseqs_db(mmseqdb, prt_path, logmmseq) # except KeyboardInterrupt: # pragma: no cover except: # pragma: no cover stop_bar = True x.join() sys.exit(1) # Clustering done, stop bar and join (if quiet, it was already finished, so we just join it) stop_bar = True x.join() return res
def documents_to_corpus(path_in: pathlib.Path, path_out: pathlib.Path) -> None: u.assert_folder_is_readable(path_in) u.assert_folder_is_writable(path_out) i = 1 widgets = [ 'Formatting Document # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker = '.', left = '[', right = ']')] with pb.ProgressBar(widgets = widgets) as bar: for file_name in path_in.iterdir(): if u.is_corpus_document(file_name): bar.update(i) i = i + 1 sentences = __tokenize_document(file_name) u.write_document(path_out, file_name, sentences)
def progress_overlay(items: t.Iterator, title: str) -> t.Iterator: bar_i = 0 widgets = [ title, ' ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker='.', left='[', right=']') ] with pb.ProgressBar(widgets=widgets) as bar: for item in items: bar_i = bar_i + 1 bar.update(bar_i) yield item
def test_no_fill(monkeypatch): '''Simply bounce within the terminal width''' bar = progressbar.BouncingBar() bar.INTERVAL = timedelta(seconds=1) p = progressbar.ProgressBar(widgets=[bar], max_value=progressbar.UnknownLength, term_width=20) assert p.term_width is not None for i in range(30): p.update(i, force=True) # Fake the start time so we can actually emulate a moving progress bar p.start_time = p.start_time - timedelta(seconds=i)
def _progress(items: t.Iterator) -> t.Iterator: bar_i = 0 widgets = [ 'Saving Items # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker='.', left='[', right=']') ] with pb.ProgressBar(widgets=widgets) as bar: for item in items: bar_i = bar_i + 1 bar.update(bar_i) yield item
def vectorize_corpus(path_in: pathlib.Path, path_out: pathlib.Path, path_control: pathlib.Path) -> None: u.assert_folder_is_readable(path_in) u.assert_folder_is_writable(path_out) i = 1 widgets = [ 'Collecting Tokens # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker = '.', left = '[', right = ']')] tokens = dict() with pb.ProgressBar(widgets = widgets) as bar: for file_name in path_in.iterdir(): if u.is_corpus_document(file_name): bar.update(i) i = i + 1 __append_tokens(tokens, file_name) token_map = __map_tokens(tokens) __save_token_file(tokens, token_map, path_control) i = 1 widgets = [ 'Vectorizing Document # ', pb.Counter(), ' ', pb.Timer(), ' ', pb.BouncingBar(marker = '.', left = '[', right = ']')] with pb.ProgressBar(widgets = widgets) as bar: for file_name in path_in.iterdir(): if u.is_corpus_document(file_name): bar.update(i) i = i + 1 vector = __vectorise_document(file_name, token_map) u.write_document(path_out, file_name, vector)
def example23(): """ Display progress bar using tqdm. >>> example23() True """ widgets = [progressbar.BouncingBar(marker=progressbar.RotatingMarker())] with progressbar.ProgressBar(widgets=widgets, max_value=20, term_width=10) as progress: for i in range(20): sleep(0.1) progress.update(i) widgets = [ progressbar.BouncingBar(marker=progressbar.RotatingMarker(), fill_left=False) ] with progressbar.ProgressBar(widgets=widgets, max_value=20, term_width=10) as progress: for i in range(20): sleep(0.1) progress.update(i) return True
def example16(): """ Display progress bar using tqdm. >>> example16() True """ widgets = [ progressbar.FormatLabel("Bouncer: value %(value)d - "), progressbar.BouncingBar() ] pbar = progressbar.ProgressBar(widgets=widgets) for i in pbar((i for i in range(25))): sleep(0.2) return True
def upload_to_solr(filename): """Commit a CSV file to Solr using a stream""" prog_widgets = [ 'Uploading... ', progressbar.BouncingBar(), ' ', progressbar.Timer(format='Time: %s') ] prog_bar = progressbar.ProgressBar(widgets=prog_widgets) prog_bar.start() # Build the Solr upload URL url = ( '"{server}/update?stream.file={path}&stream.contentType=text/csv;charset=utf-8&commit=true"' .format(server=settings.SOLR_SERVER, path=os.path.abspath(filename))) command = 'curl -s -o /dev/null -w "%{http_code}" ' + url proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) polls = 0 while proc.returncode is None: proc.poll() polls += 1 prog_bar.update((polls * UPLOAD_PROGRESS_STEP) % prog_bar.maxval) time.sleep(UPLOAD_POLL_WAIT_SECS) prog_bar.finish() if proc.returncode != 0: failure_message = 'process returned {}'.format(proc.returncode) else: status = proc.communicate()[0] if status[0] != '2': failure_message = 'status {}'.format(status) else: failure_message = None if failure_message is not None: print 'Upload failed ({}). See the Solr logs for details.'.format( failure_message) else: print 'Upload successful.'