def get_author_posts_from_all_pages(threadid, print_progress=True):
"""Return a list of post text written by the thread's author
across all pages in the thread
"""
author_name = None
posts = []
pagenum = 1 #Used only for progressbar
url = get_url_with_suffix(threadid)
if print_progress:
widgets = ['Getting Page ', progressbar.Counter(), ': ',
progressbar.AnimatedMarker(markers='v-^-'),
progressbar.Timer(),
progressbar.AnimatedMarker(markers='^-V-')]
pbar = progressbar.ProgressBar(widgets=widgets)
pbar.update(pagenum)
pbar.start()
while True:
if print_progress:
pbar.update(pagenum)
#TODO: Requests timeout parameter + check for failure
html = requests.get(url).text
soup = BeautifulSoup(html, "lxml")
if author_name is None:
author_name = get_author_name(soup)
posts += get_author_posts(soup, author_name)
next_button = soup.find(class_="pagination_next")
if next_button is None:
break
else:
url = eagletime_url + "/" + next_button['href']
pagenum += 1
return posts
def __init__(self, _min, _max):
self._min = _min
self._indeterminate = _max is None
widgets = [
'Frame ',
progressbar.Counter(),
' ',
progressbar.Percentage(),
' ',
progressbar.Timer(),
' ',
progressbar.Bar(initial_value=_min), # Placeholder
' ',
progressbar.AdaptiveTransferSpeed(unit='frames'),
' ',
progressbar.AdaptiveETA()
]
self._bar_index = 5
if self._indeterminate:
widgets[self._bar_index] = progressbar.AnimatedMarker()
self.pbar = progressbar.ProgressBar(
max_value=progressbar.UnknownLength, widgets=widgets)
else:
if _min > 0:
# Start indeterminate and switch when we reach min
widgets[self._bar_index] = progressbar.AnimatedMarker()
self.pbar = progressbar.ProgressBar(max_value=_max,
widgets=widgets)
self.pbar.start()
def generate():
bar = progressbar.ProgressBar(widgets=[
' [',
progressbar.Timer(),
'] ',
progressbar.AnimatedMarker(markers='<3 '),
progressbar.AnimatedMarker(markers='3 <'),
progressbar.AnimatedMarker(markers=' <3'),
progressbar.AnimatedMarker(markers='<3 '),
progressbar.AnimatedMarker(markers='3 <'),
progressbar.AnimatedMarker(markers=' <3'),
progressbar.AnimatedMarker(markers='<3 '),
progressbar.AnimatedMarker(markers='3 <'),
progressbar.AnimatedMarker(markers=' <3'),
progressbar.Bar(),
])
for i in range(TRANSACTIONS_NUMBER):
interval = numpy.random.exponential(1 / LAMBDA)
system_1_transactions_intervals.append(interval)
yield env.timeout(interval)
env.process(Transaction.run())
if i % (TRANSACTIONS_NUMBER / 100) == 0:
bar.update(i / (TRANSACTIONS_NUMBER / 100))
def start(self):
tasks = []
for world in self.worlds:
files = self.worlds[world].files.all()
for mcFile in files:
tasks.append(self.WorldFileTask(world, mcFile))
logger.debug("Prepare queue with Scanning Tasks %s", len(tasks))
# self.progress = progressbar.ProgressBar(min_val=1, max_val=len(tasks))
self.progress = progressbar.ProgressBar(
# widgets=[progressbar.SimpleProgress()],
widgets=[
progressbar.AnimatedMarker(),
" ",
progressbar.FormatLabel(
"Processed: %(value)d of: %(max_value)d "),
"Percent: ",
progressbar.Percentage(),
" ",
progressbar.ETA(),
" ",
progressbar.Bar(">"),
],
redirect_stdout=True,
max_value=len(tasks),
).start()
self.multiprocess_manager.append_tasks(tasks)
self.multiprocess_manager.queue.join()
self.progress.finish()
logger.debug("Finished %s", len(tasks))
for task in self.scannd_results:
logger.debug("Scanned %s", task.scanned)
def _track_progress(label: str, f: Callable):
widgets = [
termcol.COLORS.INFORMATION, label, ' ',
progressbar.AnimatedMarker(), termcol.COLORS.ENDC
]
p = progressbar.ProgressBar(widgets=widgets,
maxval=progressbar.UnknownLength,
redirect_stdout=True)
p.start()
result = []
thread = threading.Thread(target=_run_safe, args=(f, result))
thread.start()
for progress_count in itertools.count():
if not thread.is_alive():
break
p.update(progress_count)
time.sleep(0.1)
if len(result) > 0:
p.widgets = [termcol.error(label + ' ❌')]
else:
p.widgets = [termcol.success(label + ' ✓')]
p.update(force=True)
p.finish()
for r in result:
if isinstance(r, Exception):
raise r
def test_batch(self, sess=None, epoch=None):
if sess == None:
raise NotImplementedError('No session found for evaluation!')
print("Testing [%d/%d] Triples" % (self.n_test, len(self.eval_data)))
size_per_batch = self.model.config.batch_size_testing
head_rank, tail_rank = self.model.test_batch()
widgets = ['Inferring for Evaluation: ', progressbar.AnimatedMarker(), " Done:",
progressbar.Percentage(), " ", progressbar.AdaptiveETA()]
self.result_queue.put("Start!")
with progressbar.ProgressBar(max_value=self.loop_len, widgets=widgets) as bar:
for i in range(self.loop_len):
data = np.asarray([[self.eval_data[x].h, self.eval_data[x].r, self.eval_data[x].t]
for x in range(size_per_batch * i, size_per_batch * (i + 1))])
h = data[:, 0]
r = data[:, 1]
t = data[:, 2]
feed_dict = {
self.model.test_h_batch: h,
self.model.test_r_batch: r,
self.model.test_t_batch: t}
head_tmp, tail_tmp = np.squeeze(sess.run([head_rank, tail_rank], feed_dict))
result_data = [tail_tmp, head_tmp, h, r, t, epoch]
self.result_queue.put(result_data)
bar.update(i)
self.result_queue.put("Stop!")
def __unknown_length_widgets():
widgets = [
progressbar.Timer('%(elapsed)s'), ' ', '(',
progressbar.Counter(), ') ',
progressbar.AnimatedMarker()
]
return widgets
def REFRESH():
print('Refreshing...')
news = []
try:
log = pd.read_csv('log.csv', encoding='utf-8')
except FileNotFoundError:
print('The list is empty.\n')
return
widgets = [
pb.AnimatedMarker(markers='|\\-/'), ' ', 'Progress: ',
pb.Bar(marker='=', left='[', right=']'), ' ',
pb.Counter(), ' ',
pb.Percentage(), ' ',
pb.AdaptiveETA()
]
pbar = pb.ProgressBar(widgets=widgets, maxval=len(log)).start()
for x in range(0, len(log)):
latest_work, post_time = SEARCH(log.iloc[x, 0])
if post_time != log.iloc[x, 2]:
log.iloc[x, 1] = latest_work
log.iloc[x, 2] = post_time
news.append(log.iloc[x, 0] + ' - ' + post_time)
pbar.update(x + 1)
log.to_csv('log.csv', index=False, encoding='utf-8')
print('\n%-2d new work(s) found!\n' % len(news) + '-' * 20)
for x in news:
print(x)
print('-' * 20)
return
def test_all_widgets_max_width(max_width, term_width):
widgets = [
progressbar.Timer(max_width=max_width),
progressbar.ETA(max_width=max_width),
progressbar.AdaptiveETA(max_width=max_width),
progressbar.AbsoluteETA(max_width=max_width),
progressbar.DataSize(max_width=max_width),
progressbar.FileTransferSpeed(max_width=max_width),
progressbar.AdaptiveTransferSpeed(max_width=max_width),
progressbar.AnimatedMarker(max_width=max_width),
progressbar.Counter(max_width=max_width),
progressbar.Percentage(max_width=max_width),
progressbar.FormatLabel('%(value)d', max_width=max_width),
progressbar.SimpleProgress(max_width=max_width),
progressbar.Bar(max_width=max_width),
progressbar.ReverseBar(max_width=max_width),
progressbar.BouncingBar(max_width=max_width),
progressbar.FormatCustomText('Custom %(text)s',
dict(text='text'),
max_width=max_width),
progressbar.DynamicMessage('custom', max_width=max_width),
progressbar.CurrentTime(max_width=max_width),
]
p = progressbar.ProgressBar(widgets=widgets, term_width=term_width)
p.update(0)
p.update()
for widget in p._format_widgets():
if max_width and max_width < term_width:
assert widget == ''
else:
assert widget != ''
def _MakeProgressBar(self, max_size, name, message=None):
"""Returns a ProgressBar object with default widgets.
Args:
max_size (int): the size of the source.
name (str): the name of what is being processed.
message (str): an extra message to display before the bar.
Returns:
ProgressBar: the progress bar object.
"""
if message:
self._logger.info(message)
if max_size > 0:
pb = BaBar(maxval=max_size,
widgets=[
name,
progressbar.Percentage(), ' ',
progressbar.Bar('=', '[', ']'), ' ',
progressbar.ETA(),
progressbar.FileTransferSpeed()
])
else:
pb = BaBar(maxval=0, widgets=[name, progressbar.AnimatedMarker()])
return pb
def run_unbounded(self, energy_ratio_limit):
if self.data.length > 1:
last_energy, new_energy = self.calc_energy(
self.data[-2].data), self.calc_energy(self.data[-1].data)
energy_ratio = np.abs((new_energy - last_energy) / last_energy)
else:
last_energy, energy_ratio = self.calc_energy(self.data[-1].data), 1
i = self.data.length
bar = progressbar.ProgressBar(widgets=[
progressbar.AnimatedMarker(), ' ',
progressbar.Counter('%(value)05d'), ' ',
progressbar.DynamicMessage('energy_ratio'), ' ',
progressbar.Timer()
],
max_value=progressbar.UnknownLength)
while energy_ratio > energy_ratio_limit:
self.data.push((i + 1) * self.time_params.d * self.save_every)
begin, end = self.data[-2].data, self.data[-1].data
self.step_kernel(begin, end)
new_energy = self.calc_energy(end)
if new_energy > last_energy:
raise Warning('Energy has increased')
energy_ratio, last_energy = np.abs(
(new_energy - last_energy) / last_energy), new_energy
i += 1
bar.update(i * self.save_every, energy_ratio=energy_ratio)
def __init__(self, max_epoch, batch_size, N, custom_text_dict):
self.current_epoch = 0
self.current_batch = 0
self.max_batch = int(math.ceil(N / batch_size))
max_value = max_epoch * self.max_batch
base_text = '(Epoch: [%(epoch)d/%(max_epoch)d], Batch: [%(batch)d/%(max_batch)d]) '
base_dict = dict(epoch=1,
max_epoch=max_epoch,
batch=1,
max_batch=self.max_batch)
custom_text = ''
for key, value in custom_text_dict.items():
custom_text += "{} = %({}).6f, ".format(value, key)
base_dict[key] = -1
self.format_custom_text = progressbar.FormatCustomText(
base_text + custom_text,
base_dict,
)
widgets = [
progressbar.Percentage(), ' ',
progressbar.AnimatedMarker(), ' ',
progressbar.Bar(marker='█'), ' ',
progressbar.SimpleProgress(), ' ', self.format_custom_text, ' ',
progressbar.ETA()
]
self.bar = progressbar.ProgressBar(max_value=max_value,
widgets=widgets)
self.bar.start()
def cropImages():
print("extracting faces.")
for p in paths:
if not os.path.exists(p):
try:
os.mkdir(p)
except OSError:
print("error encountered while creating " + p)
widgets = [
progressbar.Timer(format='elapsed time: %(elapsed)s', ), " ",
progressbar.AnimatedMarker(markers='.oO@* ')
]
bar = progressbar.ProgressBar(widgets=widgets)
for image in os.listdir(imagePath):
processImage(os.path.join(imagePath, image), labelPath)
bar.update()
print()
print()
for p in paths[1:]:
print(p + ": " + str(len(os.listdir(p))) + " images.")
print()
print("finished extracting faces.")
print()
def test_all_widgets_small_values(max_value):
widgets = [
progressbar.Timer(),
progressbar.ETA(),
progressbar.AdaptiveETA(),
progressbar.AbsoluteETA(),
progressbar.DataSize(),
progressbar.FileTransferSpeed(),
progressbar.AdaptiveTransferSpeed(),
progressbar.AnimatedMarker(),
progressbar.Counter(),
progressbar.Percentage(),
progressbar.FormatLabel('%(value)d'),
progressbar.SimpleProgress(),
progressbar.Bar(),
progressbar.ReverseBar(),
progressbar.BouncingBar(),
progressbar.CurrentTime(),
progressbar.CurrentTime(microseconds=False),
progressbar.CurrentTime(microseconds=True),
]
p = progressbar.ProgressBar(widgets=widgets, max_value=max_value)
for i in range(10):
time.sleep(1)
p.update(i + 1)
p.finish()
def test_all_widgets_large_values(max_value):
widgets = [
progressbar.Timer(),
progressbar.ETA(),
progressbar.AdaptiveETA(),
progressbar.AbsoluteETA(),
progressbar.DataSize(),
progressbar.FileTransferSpeed(),
progressbar.AdaptiveTransferSpeed(),
progressbar.AnimatedMarker(),
progressbar.Counter(),
progressbar.Percentage(),
progressbar.FormatLabel('%(value)d/%(max_value)d'),
progressbar.SimpleProgress(),
progressbar.Bar(fill=lambda progress, data, width: '#'),
progressbar.ReverseBar(),
progressbar.BouncingBar(),
progressbar.FormatCustomText('Custom %(text)s', dict(text='text')),
]
p = progressbar.ProgressBar(widgets=widgets, max_value=max_value)
p.update()
time.sleep(1)
p.update()
for i in range(0, 10**6, 10**4):
time.sleep(1)
p.update(i)
def download_file_to(link, path):
r = requests.get(link, stream=True)
if r.status_code != 200:
print(r.status_code, link)
return False
file_size = int(r.headers['Content-length'])
print("Downloading file %s (%d bytes)" % (path, file_size))
widgets = [
'%s: ' % os.path.basename(path),
progressbar.Percentage(),
' ',
progressbar.Bar(marker=progressbar.AnimatedMarker(fill='#')),
' ',
progressbar.Counter('%(value)d'),
'/' + str(file_size) + ' bytes downloaded',
' ',
progressbar.ETA(),
' ',
progressbar.FileTransferSpeed(),
]
bar = progressbar.ProgressBar(widgets=widgets,
max_value=file_size,
redirect_stdout=True).start()
with open(path, "wb") as f:
part = 8192
for chunk in r.iter_content(part):
bar += len(chunk)
f.write(chunk)
bar.finish()
return True
def progressAnimated():
widgets = ['Requesting new identity..', progressbar.AnimatedMarker(". ")]
bar = progressbar.ProgressBar(widgets=widgets).start()
for i in range(20):
time.sleep(.5)
bar.update(i)
print(' ')
def filling_bar_animated_marker():
bar = progressbar.ProgressBar(widgets=[
progressbar.Bar(
marker=progressbar.AnimatedMarker(fill='#'),
),
])
for i in bar(range(15)):
time.sleep(0.1)
def animated_marker():
widgets = ['In Process: ', progressbar.AnimatedMarker()]
bar = progressbar.ProgressBar(widgets=widgets).start()
for i in range(17):
time.sleep(0.1)
bar.update(i)
print("\033[A \033[A")
print(colors.fg.green, "Finished!", colors.reset)
def animated_marker():
widgets = ['Loading: ', progressbar.AnimatedMarker()]
bar = progressbar.ProgressBar(widgets=widgets).start()
for i in range(15):
time.sleep(0.1)
bar.update(i)
print("Finished")
def get_mb():
print_main()
widgets = [progressbar.Percentage(),
' ', progressbar.Bar(),
' ', progressbar.ETA(),
' ', progressbar.AnimatedMarker(markers='◐◓◑◒')
]
bar = progressbar.ProgressBar(widgets=widgets, maxval=150.0).start()
t = 0.0
for i, inn in enumerate(inns_mb, start=1):
value1, value2, name_org = get_value(inn)
urls = [f'http://bus.gov.ru/public/agency/last_task.json?agency={value1}&d-5460-o=2&d-5460-s=1&task=',
f'http://bus.gov.ru/public/agency/last-agency-plan.json?agency={value1}&d-5460-o=2&d-5460-s=1',
f'http://bus.gov.ru/public/agency/last-operation.json?agency={value1}&stage=',
f'http://bus.gov.ru/public/agency/last-annual-balance-F0503721-info.json?agencyId={value1}&d-5460-o=2&d-5460-s=1',
f'http://bus.gov.ru/public/agency/last-annual-balance-F0503730-info.json?agencyId={value1}',
f'http://bus.gov.ru/public/annual-balance-f0503737/show-last-annual-balance.json?agencyId={value1}',
f'http://bus.gov.ru/public/agency/last_activity_estate_usage_report.json?agency={value1}&d-5460-o=2&d-5460-s=1',
f'http://bus.gov.ru/public/agency/last-measure-details.json?agency={value1}&d-5460-o=2&d-5460-s=1'
]
result = pars(urls, inn)
bar.update(t)
t += 10.0
buh = sum([result[3], result[4], result[5]])
if buh == 3:
s_.append(1)
pr = sum(result) * 100 / 8
get_print(i, name_org, result, pr, "+" if pr == 100.0 else "")
matrix.append(result)
bar.finish()
p1 = sum([matrix[j][0] for j in range(len(matrix))])
p2 = sum([matrix[j][1] for j in range(len(matrix))])
p3 = sum([matrix[j][2] for j in range(len(matrix))])
p5 = sum(s_)
p7 = sum([matrix[j][6] for j in range(len(matrix))])
p8 = sum([matrix[j][7] for j in range(len(matrix))])
print_end(p1, p2, p3, p5, p7, p8)
def create_progressbar_reader(reader, max_reads=None, mag_format=None):
import progressbar
import progressbar.widgets
import math
class ProgressBarReader(progressbar.ProgressBar):
def __init__(self, iterable, widgets, max_value=None):
super(ProgressBarReader,
self).__init__(widgets=widgets,
max_value=max_value
or progressbar.UnknownLength)
self._iterable = iterable
self.done = False
def __next__(self):
try:
value = next(self._iterable)
if self.start_time is None:
self.start()
self.update(self.value + value[0])
return value
except StopIteration:
self.close()
raise
def close(self):
if not self.done:
self.finish()
self.done = True
try:
self._iterable.close()
except:
pass
class MagCounter(progressbar.widgets.WidgetBase):
def __init__(self, mag_format):
self._format = mag_format
def __call__(self, progress, data):
return self._format(data["value"])
if max_reads:
reader = ProgressBarReader(reader, [
MagCounter(mag_format), " Reads (",
progressbar.Percentage(), ") ",
progressbar.Timer(), " ",
progressbar.Bar(),
progressbar.AdaptiveETA()
], max_reads)
else:
reader = ProgressBarReader(reader, [
MagCounter(mag_format), " Reads",
progressbar.Timer(),
progressbar.AnimatedMarker()
])
return reader
def get_progressbar_widgets(maximum_length):
return [
progressbar.AnimatedMarker(markers='←↑→↓'),
progressbar.Counter(
format=' Progress: %(value)d of {}'.format(maximum_length)),
progressbar.Timer(format=' Elapsed: %(elapsed)s'),
progressbar.Bar(marker='█'),
progressbar.widgets.Percentage()
]
def animated_wheels():
# You may need python 3.x to see this correctly
try:
widgets = ['Wheels: ', progressbar.AnimatedMarker(markers='◐◓◑◒')]
bar = progressbar.ProgressBar(widgets=widgets)
for i in bar((i for i in range(24))):
time.sleep(0.1)
except UnicodeError:
sys.stdout.write('Unicode error: skipping example')
def __init__(self, message="Waiting", **kwargs):
widgets = [
"%s " % message,
progressbar.AnimatedMarker(markers='.oO@* '),
progressbar.Timer(format=" %s")
]
super(Balloon, self).__init__(maxval=progressbar.UnknownLength,
widgets=widgets,
**kwargs)
self.start()
def test_init_progress_bar_with_unknown_length(self,
mock_is_dumb_terminal):
mock_is_dumb_terminal.return_value = self.dumb
pb = indicators._init_progress_bar(0, "destination", "message")
self.assertEqual(pb.maxval, progressbar.UnknownLength)
self.assertTrue("message" in pb.widgets)
pb_widgets_types = [type(w) for w in pb.widgets]
self.assertEqual(
type(progressbar.AnimatedMarker()) in pb_widgets_types,
not self.dumb)
def test_iterator_without_max_value():
'''Progressbar can't guess max_value.'''
p = progressbar.ProgressBar(widgets=[
progressbar.AnimatedMarker(),
progressbar.FormatLabel('%(value)d'),
progressbar.BouncingBar(),
progressbar.BouncingBar(marker=progressbar.RotatingMarker()),
])
for i in p((i for i in range(10))):
time.sleep(0.001)
async def extract_tarball(file_path: str, target_path: str, compression: str = None):
mode = "r|"
if compression is not None and compression in {'bz2', 'gz', 'xz'}:
mode += compression
else:
mode += "*"
_logger.info("Extracting %s to %s", file_path, target_path)
mkdirs(target_path)
speed = progressbar.AdaptiveTransferSpeed(samples=datetime.timedelta(seconds=5))
speed.INTERVAL = datetime.timedelta(milliseconds=500)
eta = progressbar.AdaptiveETA(samples=datetime.timedelta(seconds=5))
eta.INTERVAL = datetime.timedelta(milliseconds=500)
bar = progressbar.ProgressBar(widgets=[
' ',
progressbar.Timer(format='%(elapsed)s'), f' - Reading {file_path} @ ', speed, ' - ',
progressbar.AnimatedMarker()
],
redirect_stdout=True)
with bar:
bar.start()
with __open_tarball(file_path, mode) as tar:
for member in tar:
member.name = re.sub(r'[:]', '_', member.name)
if not member.isfile():
continue
fileobj = tar.extractfile(member)
if fileobj is None:
continue
rel_path = member.name.split("/")
target_file = join(target_path, *rel_path)
mkdirs(dirname(target_file))
async with aopen(target_file, "wb") as f:
while True:
chunk = fileobj.read()
if not chunk:
break
await f.write(chunk)
bar += member.size
def animated_marker():
widgets = ['Loading tools: ', progressbar.AnimatedMarker()]
bar = progressbar.ProgressBar(widgets=widgets).start()
for i in range(50):
t.sleep(0.1)
bar.update(i)
def _create_pbar(self, max_iter):
"""
Creates a progress bar.
"""
self.grad_iter = 0
self.pbar = pb.ProgressBar()
self.pbar.widgets = ["Optimizing: ", pb.Percentage(),
" ", pb.Bar(marker=pb.AnimatedMarker()),
" ", pb.ETA()]
self.pbar.maxval = max_iter
