def get_author_posts_from_all_pages(threadid, print_progress=True): """Return a list of post text written by the thread's author across all pages in the thread """ author_name = None posts = [] pagenum = 1 #Used only for progressbar url = get_url_with_suffix(threadid) if print_progress: widgets = ['Getting Page ', progressbar.Counter(), ': ', progressbar.AnimatedMarker(markers='v-^-'), progressbar.Timer(), progressbar.AnimatedMarker(markers='^-V-')] pbar = progressbar.ProgressBar(widgets=widgets) pbar.update(pagenum) pbar.start() while True: if print_progress: pbar.update(pagenum) #TODO: Requests timeout parameter + check for failure html = requests.get(url).text soup = BeautifulSoup(html, "lxml") if author_name is None: author_name = get_author_name(soup) posts += get_author_posts(soup, author_name) next_button = soup.find(class_="pagination_next") if next_button is None: break else: url = eagletime_url + "/" + next_button['href'] pagenum += 1 return posts
def __init__(self, _min, _max): self._min = _min self._indeterminate = _max is None widgets = [ 'Frame ', progressbar.Counter(), ' ', progressbar.Percentage(), ' ', progressbar.Timer(), ' ', progressbar.Bar(initial_value=_min), # Placeholder ' ', progressbar.AdaptiveTransferSpeed(unit='frames'), ' ', progressbar.AdaptiveETA() ] self._bar_index = 5 if self._indeterminate: widgets[self._bar_index] = progressbar.AnimatedMarker() self.pbar = progressbar.ProgressBar( max_value=progressbar.UnknownLength, widgets=widgets) else: if _min > 0: # Start indeterminate and switch when we reach min widgets[self._bar_index] = progressbar.AnimatedMarker() self.pbar = progressbar.ProgressBar(max_value=_max, widgets=widgets) self.pbar.start()
def generate(): bar = progressbar.ProgressBar(widgets=[ ' [', progressbar.Timer(), '] ', progressbar.AnimatedMarker(markers='<3 '), progressbar.AnimatedMarker(markers='3 <'), progressbar.AnimatedMarker(markers=' <3'), progressbar.AnimatedMarker(markers='<3 '), progressbar.AnimatedMarker(markers='3 <'), progressbar.AnimatedMarker(markers=' <3'), progressbar.AnimatedMarker(markers='<3 '), progressbar.AnimatedMarker(markers='3 <'), progressbar.AnimatedMarker(markers=' <3'), progressbar.Bar(), ]) for i in range(TRANSACTIONS_NUMBER): interval = numpy.random.exponential(1 / LAMBDA) system_1_transactions_intervals.append(interval) yield env.timeout(interval) env.process(Transaction.run()) if i % (TRANSACTIONS_NUMBER / 100) == 0: bar.update(i / (TRANSACTIONS_NUMBER / 100))
def start(self): tasks = [] for world in self.worlds: files = self.worlds[world].files.all() for mcFile in files: tasks.append(self.WorldFileTask(world, mcFile)) logger.debug("Prepare queue with Scanning Tasks %s", len(tasks)) # self.progress = progressbar.ProgressBar(min_val=1, max_val=len(tasks)) self.progress = progressbar.ProgressBar( # widgets=[progressbar.SimpleProgress()], widgets=[ progressbar.AnimatedMarker(), " ", progressbar.FormatLabel( "Processed: %(value)d of: %(max_value)d "), "Percent: ", progressbar.Percentage(), " ", progressbar.ETA(), " ", progressbar.Bar(">"), ], redirect_stdout=True, max_value=len(tasks), ).start() self.multiprocess_manager.append_tasks(tasks) self.multiprocess_manager.queue.join() self.progress.finish() logger.debug("Finished %s", len(tasks)) for task in self.scannd_results: logger.debug("Scanned %s", task.scanned)
def _track_progress(label: str, f: Callable): widgets = [ termcol.COLORS.INFORMATION, label, ' ', progressbar.AnimatedMarker(), termcol.COLORS.ENDC ] p = progressbar.ProgressBar(widgets=widgets, maxval=progressbar.UnknownLength, redirect_stdout=True) p.start() result = [] thread = threading.Thread(target=_run_safe, args=(f, result)) thread.start() for progress_count in itertools.count(): if not thread.is_alive(): break p.update(progress_count) time.sleep(0.1) if len(result) > 0: p.widgets = [termcol.error(label + ' ❌')] else: p.widgets = [termcol.success(label + ' ✓')] p.update(force=True) p.finish() for r in result: if isinstance(r, Exception): raise r
def test_batch(self, sess=None, epoch=None): if sess == None: raise NotImplementedError('No session found for evaluation!') print("Testing [%d/%d] Triples" % (self.n_test, len(self.eval_data))) size_per_batch = self.model.config.batch_size_testing head_rank, tail_rank = self.model.test_batch() widgets = ['Inferring for Evaluation: ', progressbar.AnimatedMarker(), " Done:", progressbar.Percentage(), " ", progressbar.AdaptiveETA()] self.result_queue.put("Start!") with progressbar.ProgressBar(max_value=self.loop_len, widgets=widgets) as bar: for i in range(self.loop_len): data = np.asarray([[self.eval_data[x].h, self.eval_data[x].r, self.eval_data[x].t] for x in range(size_per_batch * i, size_per_batch * (i + 1))]) h = data[:, 0] r = data[:, 1] t = data[:, 2] feed_dict = { self.model.test_h_batch: h, self.model.test_r_batch: r, self.model.test_t_batch: t} head_tmp, tail_tmp = np.squeeze(sess.run([head_rank, tail_rank], feed_dict)) result_data = [tail_tmp, head_tmp, h, r, t, epoch] self.result_queue.put(result_data) bar.update(i) self.result_queue.put("Stop!")
def __unknown_length_widgets(): widgets = [ progressbar.Timer('%(elapsed)s'), ' ', '(', progressbar.Counter(), ') ', progressbar.AnimatedMarker() ] return widgets
def REFRESH(): print('Refreshing...') news = [] try: log = pd.read_csv('log.csv', encoding='utf-8') except FileNotFoundError: print('The list is empty.\n') return widgets = [ pb.AnimatedMarker(markers='|\\-/'), ' ', 'Progress: ', pb.Bar(marker='=', left='[', right=']'), ' ', pb.Counter(), ' ', pb.Percentage(), ' ', pb.AdaptiveETA() ] pbar = pb.ProgressBar(widgets=widgets, maxval=len(log)).start() for x in range(0, len(log)): latest_work, post_time = SEARCH(log.iloc[x, 0]) if post_time != log.iloc[x, 2]: log.iloc[x, 1] = latest_work log.iloc[x, 2] = post_time news.append(log.iloc[x, 0] + ' - ' + post_time) pbar.update(x + 1) log.to_csv('log.csv', index=False, encoding='utf-8') print('\n%-2d new work(s) found!\n' % len(news) + '-' * 20) for x in news: print(x) print('-' * 20) return
def test_all_widgets_max_width(max_width, term_width): widgets = [ progressbar.Timer(max_width=max_width), progressbar.ETA(max_width=max_width), progressbar.AdaptiveETA(max_width=max_width), progressbar.AbsoluteETA(max_width=max_width), progressbar.DataSize(max_width=max_width), progressbar.FileTransferSpeed(max_width=max_width), progressbar.AdaptiveTransferSpeed(max_width=max_width), progressbar.AnimatedMarker(max_width=max_width), progressbar.Counter(max_width=max_width), progressbar.Percentage(max_width=max_width), progressbar.FormatLabel('%(value)d', max_width=max_width), progressbar.SimpleProgress(max_width=max_width), progressbar.Bar(max_width=max_width), progressbar.ReverseBar(max_width=max_width), progressbar.BouncingBar(max_width=max_width), progressbar.FormatCustomText('Custom %(text)s', dict(text='text'), max_width=max_width), progressbar.DynamicMessage('custom', max_width=max_width), progressbar.CurrentTime(max_width=max_width), ] p = progressbar.ProgressBar(widgets=widgets, term_width=term_width) p.update(0) p.update() for widget in p._format_widgets(): if max_width and max_width < term_width: assert widget == '' else: assert widget != ''
def _MakeProgressBar(self, max_size, name, message=None): """Returns a ProgressBar object with default widgets. Args: max_size (int): the size of the source. name (str): the name of what is being processed. message (str): an extra message to display before the bar. Returns: ProgressBar: the progress bar object. """ if message: self._logger.info(message) if max_size > 0: pb = BaBar(maxval=max_size, widgets=[ name, progressbar.Percentage(), ' ', progressbar.Bar('=', '[', ']'), ' ', progressbar.ETA(), progressbar.FileTransferSpeed() ]) else: pb = BaBar(maxval=0, widgets=[name, progressbar.AnimatedMarker()]) return pb
def run_unbounded(self, energy_ratio_limit): if self.data.length > 1: last_energy, new_energy = self.calc_energy( self.data[-2].data), self.calc_energy(self.data[-1].data) energy_ratio = np.abs((new_energy - last_energy) / last_energy) else: last_energy, energy_ratio = self.calc_energy(self.data[-1].data), 1 i = self.data.length bar = progressbar.ProgressBar(widgets=[ progressbar.AnimatedMarker(), ' ', progressbar.Counter('%(value)05d'), ' ', progressbar.DynamicMessage('energy_ratio'), ' ', progressbar.Timer() ], max_value=progressbar.UnknownLength) while energy_ratio > energy_ratio_limit: self.data.push((i + 1) * self.time_params.d * self.save_every) begin, end = self.data[-2].data, self.data[-1].data self.step_kernel(begin, end) new_energy = self.calc_energy(end) if new_energy > last_energy: raise Warning('Energy has increased') energy_ratio, last_energy = np.abs( (new_energy - last_energy) / last_energy), new_energy i += 1 bar.update(i * self.save_every, energy_ratio=energy_ratio)
def __init__(self, max_epoch, batch_size, N, custom_text_dict): self.current_epoch = 0 self.current_batch = 0 self.max_batch = int(math.ceil(N / batch_size)) max_value = max_epoch * self.max_batch base_text = '(Epoch: [%(epoch)d/%(max_epoch)d], Batch: [%(batch)d/%(max_batch)d]) ' base_dict = dict(epoch=1, max_epoch=max_epoch, batch=1, max_batch=self.max_batch) custom_text = '' for key, value in custom_text_dict.items(): custom_text += "{} = %({}).6f, ".format(value, key) base_dict[key] = -1 self.format_custom_text = progressbar.FormatCustomText( base_text + custom_text, base_dict, ) widgets = [ progressbar.Percentage(), ' ', progressbar.AnimatedMarker(), ' ', progressbar.Bar(marker='█'), ' ', progressbar.SimpleProgress(), ' ', self.format_custom_text, ' ', progressbar.ETA() ] self.bar = progressbar.ProgressBar(max_value=max_value, widgets=widgets) self.bar.start()
def cropImages(): print("extracting faces.") for p in paths: if not os.path.exists(p): try: os.mkdir(p) except OSError: print("error encountered while creating " + p) widgets = [ progressbar.Timer(format='elapsed time: %(elapsed)s', ), " ", progressbar.AnimatedMarker(markers='.oO@* ') ] bar = progressbar.ProgressBar(widgets=widgets) for image in os.listdir(imagePath): processImage(os.path.join(imagePath, image), labelPath) bar.update() print() print() for p in paths[1:]: print(p + ": " + str(len(os.listdir(p))) + " images.") print() print("finished extracting faces.") print()
def test_all_widgets_small_values(max_value): widgets = [ progressbar.Timer(), progressbar.ETA(), progressbar.AdaptiveETA(), progressbar.AbsoluteETA(), progressbar.DataSize(), progressbar.FileTransferSpeed(), progressbar.AdaptiveTransferSpeed(), progressbar.AnimatedMarker(), progressbar.Counter(), progressbar.Percentage(), progressbar.FormatLabel('%(value)d'), progressbar.SimpleProgress(), progressbar.Bar(), progressbar.ReverseBar(), progressbar.BouncingBar(), progressbar.CurrentTime(), progressbar.CurrentTime(microseconds=False), progressbar.CurrentTime(microseconds=True), ] p = progressbar.ProgressBar(widgets=widgets, max_value=max_value) for i in range(10): time.sleep(1) p.update(i + 1) p.finish()
def test_all_widgets_large_values(max_value): widgets = [ progressbar.Timer(), progressbar.ETA(), progressbar.AdaptiveETA(), progressbar.AbsoluteETA(), progressbar.DataSize(), progressbar.FileTransferSpeed(), progressbar.AdaptiveTransferSpeed(), progressbar.AnimatedMarker(), progressbar.Counter(), progressbar.Percentage(), progressbar.FormatLabel('%(value)d/%(max_value)d'), progressbar.SimpleProgress(), progressbar.Bar(fill=lambda progress, data, width: '#'), progressbar.ReverseBar(), progressbar.BouncingBar(), progressbar.FormatCustomText('Custom %(text)s', dict(text='text')), ] p = progressbar.ProgressBar(widgets=widgets, max_value=max_value) p.update() time.sleep(1) p.update() for i in range(0, 10**6, 10**4): time.sleep(1) p.update(i)
def download_file_to(link, path): r = requests.get(link, stream=True) if r.status_code != 200: print(r.status_code, link) return False file_size = int(r.headers['Content-length']) print("Downloading file %s (%d bytes)" % (path, file_size)) widgets = [ '%s: ' % os.path.basename(path), progressbar.Percentage(), ' ', progressbar.Bar(marker=progressbar.AnimatedMarker(fill='#')), ' ', progressbar.Counter('%(value)d'), '/' + str(file_size) + ' bytes downloaded', ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed(), ] bar = progressbar.ProgressBar(widgets=widgets, max_value=file_size, redirect_stdout=True).start() with open(path, "wb") as f: part = 8192 for chunk in r.iter_content(part): bar += len(chunk) f.write(chunk) bar.finish() return True
def progressAnimated(): widgets = ['Requesting new identity..', progressbar.AnimatedMarker(". ")] bar = progressbar.ProgressBar(widgets=widgets).start() for i in range(20): time.sleep(.5) bar.update(i) print(' ')
def filling_bar_animated_marker(): bar = progressbar.ProgressBar(widgets=[ progressbar.Bar( marker=progressbar.AnimatedMarker(fill='#'), ), ]) for i in bar(range(15)): time.sleep(0.1)
def animated_marker(): widgets = ['In Process: ', progressbar.AnimatedMarker()] bar = progressbar.ProgressBar(widgets=widgets).start() for i in range(17): time.sleep(0.1) bar.update(i) print("\033[A \033[A") print(colors.fg.green, "Finished!", colors.reset)
def animated_marker(): widgets = ['Loading: ', progressbar.AnimatedMarker()] bar = progressbar.ProgressBar(widgets=widgets).start() for i in range(15): time.sleep(0.1) bar.update(i) print("Finished")
def get_mb(): print_main() widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA(), ' ', progressbar.AnimatedMarker(markers='◐◓◑◒') ] bar = progressbar.ProgressBar(widgets=widgets, maxval=150.0).start() t = 0.0 for i, inn in enumerate(inns_mb, start=1): value1, value2, name_org = get_value(inn) urls = [f'http://bus.gov.ru/public/agency/last_task.json?agency={value1}&d-5460-o=2&d-5460-s=1&task=', f'http://bus.gov.ru/public/agency/last-agency-plan.json?agency={value1}&d-5460-o=2&d-5460-s=1', f'http://bus.gov.ru/public/agency/last-operation.json?agency={value1}&stage=', f'http://bus.gov.ru/public/agency/last-annual-balance-F0503721-info.json?agencyId={value1}&d-5460-o=2&d-5460-s=1', f'http://bus.gov.ru/public/agency/last-annual-balance-F0503730-info.json?agencyId={value1}', f'http://bus.gov.ru/public/annual-balance-f0503737/show-last-annual-balance.json?agencyId={value1}', f'http://bus.gov.ru/public/agency/last_activity_estate_usage_report.json?agency={value1}&d-5460-o=2&d-5460-s=1', f'http://bus.gov.ru/public/agency/last-measure-details.json?agency={value1}&d-5460-o=2&d-5460-s=1' ] result = pars(urls, inn) bar.update(t) t += 10.0 buh = sum([result[3], result[4], result[5]]) if buh == 3: s_.append(1) pr = sum(result) * 100 / 8 get_print(i, name_org, result, pr, "+" if pr == 100.0 else "") matrix.append(result) bar.finish() p1 = sum([matrix[j][0] for j in range(len(matrix))]) p2 = sum([matrix[j][1] for j in range(len(matrix))]) p3 = sum([matrix[j][2] for j in range(len(matrix))]) p5 = sum(s_) p7 = sum([matrix[j][6] for j in range(len(matrix))]) p8 = sum([matrix[j][7] for j in range(len(matrix))]) print_end(p1, p2, p3, p5, p7, p8)
def create_progressbar_reader(reader, max_reads=None, mag_format=None): import progressbar import progressbar.widgets import math class ProgressBarReader(progressbar.ProgressBar): def __init__(self, iterable, widgets, max_value=None): super(ProgressBarReader, self).__init__(widgets=widgets, max_value=max_value or progressbar.UnknownLength) self._iterable = iterable self.done = False def __next__(self): try: value = next(self._iterable) if self.start_time is None: self.start() self.update(self.value + value[0]) return value except StopIteration: self.close() raise def close(self): if not self.done: self.finish() self.done = True try: self._iterable.close() except: pass class MagCounter(progressbar.widgets.WidgetBase): def __init__(self, mag_format): self._format = mag_format def __call__(self, progress, data): return self._format(data["value"]) if max_reads: reader = ProgressBarReader(reader, [ MagCounter(mag_format), " Reads (", progressbar.Percentage(), ") ", progressbar.Timer(), " ", progressbar.Bar(), progressbar.AdaptiveETA() ], max_reads) else: reader = ProgressBarReader(reader, [ MagCounter(mag_format), " Reads", progressbar.Timer(), progressbar.AnimatedMarker() ]) return reader
def get_progressbar_widgets(maximum_length): return [ progressbar.AnimatedMarker(markers='←↑→↓'), progressbar.Counter( format=' Progress: %(value)d of {}'.format(maximum_length)), progressbar.Timer(format=' Elapsed: %(elapsed)s'), progressbar.Bar(marker='█'), progressbar.widgets.Percentage() ]
def animated_wheels(): # You may need python 3.x to see this correctly try: widgets = ['Wheels: ', progressbar.AnimatedMarker(markers='◐◓◑◒')] bar = progressbar.ProgressBar(widgets=widgets) for i in bar((i for i in range(24))): time.sleep(0.1) except UnicodeError: sys.stdout.write('Unicode error: skipping example')
def __init__(self, message="Waiting", **kwargs): widgets = [ "%s " % message, progressbar.AnimatedMarker(markers='.oO@* '), progressbar.Timer(format=" %s") ] super(Balloon, self).__init__(maxval=progressbar.UnknownLength, widgets=widgets, **kwargs) self.start()
def test_init_progress_bar_with_unknown_length(self, mock_is_dumb_terminal): mock_is_dumb_terminal.return_value = self.dumb pb = indicators._init_progress_bar(0, "destination", "message") self.assertEqual(pb.maxval, progressbar.UnknownLength) self.assertTrue("message" in pb.widgets) pb_widgets_types = [type(w) for w in pb.widgets] self.assertEqual( type(progressbar.AnimatedMarker()) in pb_widgets_types, not self.dumb)
def test_iterator_without_max_value(): '''Progressbar can't guess max_value.''' p = progressbar.ProgressBar(widgets=[ progressbar.AnimatedMarker(), progressbar.FormatLabel('%(value)d'), progressbar.BouncingBar(), progressbar.BouncingBar(marker=progressbar.RotatingMarker()), ]) for i in p((i for i in range(10))): time.sleep(0.001)
async def extract_tarball(file_path: str, target_path: str, compression: str = None): mode = "r|" if compression is not None and compression in {'bz2', 'gz', 'xz'}: mode += compression else: mode += "*" _logger.info("Extracting %s to %s", file_path, target_path) mkdirs(target_path) speed = progressbar.AdaptiveTransferSpeed(samples=datetime.timedelta(seconds=5)) speed.INTERVAL = datetime.timedelta(milliseconds=500) eta = progressbar.AdaptiveETA(samples=datetime.timedelta(seconds=5)) eta.INTERVAL = datetime.timedelta(milliseconds=500) bar = progressbar.ProgressBar(widgets=[ ' ', progressbar.Timer(format='%(elapsed)s'), f' - Reading {file_path} @ ', speed, ' - ', progressbar.AnimatedMarker() ], redirect_stdout=True) with bar: bar.start() with __open_tarball(file_path, mode) as tar: for member in tar: member.name = re.sub(r'[:]', '_', member.name) if not member.isfile(): continue fileobj = tar.extractfile(member) if fileobj is None: continue rel_path = member.name.split("/") target_file = join(target_path, *rel_path) mkdirs(dirname(target_file)) async with aopen(target_file, "wb") as f: while True: chunk = fileobj.read() if not chunk: break await f.write(chunk) bar += member.size
def animated_marker(): widgets = ['Loading tools: ', progressbar.AnimatedMarker()] bar = progressbar.ProgressBar(widgets=widgets).start() for i in range(50): t.sleep(0.1) bar.update(i)
def _create_pbar(self, max_iter): """ Creates a progress bar. """ self.grad_iter = 0 self.pbar = pb.ProgressBar() self.pbar.widgets = ["Optimizing: ", pb.Percentage(), " ", pb.Bar(marker=pb.AnimatedMarker()), " ", pb.ETA()] self.pbar.maxval = max_iter