def upload_resumable(drive, to_upload, parent): filesize = to_upload.stat().st_size media = MediaFileUpload(to_upload, resumable=True, chunksize=1024 * 1024) request = drive.files().create(media_body=media, supportsAllDrives=True, body={ 'name': to_upload.name, 'parents': [parent], 'modifiedTime': get_time_rfc3339(to_upload) }) response = None with progressbar.ProgressBar(max_value=filesize, widgets=[ f'uploading {to_upload.as_posix()} ', progressbar.AdaptiveTransferSpeed(), ' ', progressbar.Bar(), ' ', progressbar.AdaptiveETA() ]) as pbar: while response is None: status, response = request.next_chunk() if status: pbar.update(status.resumable_progress) return response
def test_all_widgets_max_width(max_width, term_width): widgets = [ progressbar.Timer(max_width=max_width), progressbar.ETA(max_width=max_width), progressbar.AdaptiveETA(max_width=max_width), progressbar.AbsoluteETA(max_width=max_width), progressbar.DataSize(max_width=max_width), progressbar.FileTransferSpeed(max_width=max_width), progressbar.AdaptiveTransferSpeed(max_width=max_width), progressbar.AnimatedMarker(max_width=max_width), progressbar.Counter(max_width=max_width), progressbar.Percentage(max_width=max_width), progressbar.FormatLabel('%(value)d', max_width=max_width), progressbar.SimpleProgress(max_width=max_width), progressbar.Bar(max_width=max_width), progressbar.ReverseBar(max_width=max_width), progressbar.BouncingBar(max_width=max_width), progressbar.FormatCustomText('Custom %(text)s', dict(text='text'), max_width=max_width), progressbar.DynamicMessage('custom', max_width=max_width), progressbar.CurrentTime(max_width=max_width), ] p = progressbar.ProgressBar(widgets=widgets, term_width=term_width) p.update(0) p.update() for widget in p._format_widgets(): if max_width and max_width < term_width: assert widget == '' else: assert widget != ''
def eta_types_demonstration(): widgets = [ progressbar.Percentage(), ' ETA: ', progressbar.ETA(), ' Adaptive ETA: ', progressbar.AdaptiveETA(), ' Absolute ETA: ', progressbar.AbsoluteETA(), ' Transfer Speed: ', progressbar.FileTransferSpeed(), ' Adaptive Transfer Speed: ', progressbar.AdaptiveTransferSpeed(), ' ', progressbar.Bar(), ] bar = progressbar.ProgressBar(widgets=widgets, max_value=500) bar.start() for i in range(500): if i < 100: time.sleep(0.02) elif i > 400: time.sleep(0.1) else: time.sleep(0.01) bar.update(i + 1) bar.finish()
def test_all_widgets_small_values(max_value): widgets = [ progressbar.Timer(), progressbar.ETA(), progressbar.AdaptiveETA(), progressbar.AbsoluteETA(), progressbar.DataSize(), progressbar.FileTransferSpeed(), progressbar.AdaptiveTransferSpeed(), progressbar.AnimatedMarker(), progressbar.Counter(), progressbar.Percentage(), progressbar.FormatLabel('%(value)d'), progressbar.SimpleProgress(), progressbar.Bar(), progressbar.ReverseBar(), progressbar.BouncingBar(), progressbar.CurrentTime(), progressbar.CurrentTime(microseconds=False), progressbar.CurrentTime(microseconds=True), ] p = progressbar.ProgressBar(widgets=widgets, max_value=max_value) for i in range(10): time.sleep(1) p.update(i + 1) p.finish()
def test_all_widgets_large_values(max_value): widgets = [ progressbar.Timer(), progressbar.ETA(), progressbar.AdaptiveETA(), progressbar.AbsoluteETA(), progressbar.DataSize(), progressbar.FileTransferSpeed(), progressbar.AdaptiveTransferSpeed(), progressbar.AnimatedMarker(), progressbar.Counter(), progressbar.Percentage(), progressbar.FormatLabel('%(value)d/%(max_value)d'), progressbar.SimpleProgress(), progressbar.Bar(fill=lambda progress, data, width: '#'), progressbar.ReverseBar(), progressbar.BouncingBar(), progressbar.FormatCustomText('Custom %(text)s', dict(text='text')), ] p = progressbar.ProgressBar(widgets=widgets, max_value=max_value) p.update() time.sleep(1) p.update() for i in range(0, 10**6, 10**4): time.sleep(1) p.update(i)
def __init__(self, _min, _max): self._min = _min self._indeterminate = _max is None widgets = [ 'Frame ', progressbar.Counter(), ' ', progressbar.Percentage(), ' ', progressbar.Timer(), ' ', progressbar.Bar(initial_value=_min), # Placeholder ' ', progressbar.AdaptiveTransferSpeed(unit='frames'), ' ', progressbar.AdaptiveETA() ] self._bar_index = 5 if self._indeterminate: widgets[self._bar_index] = progressbar.AnimatedMarker() self.pbar = progressbar.ProgressBar( max_value=progressbar.UnknownLength, widgets=widgets) else: if _min > 0: # Start indeterminate and switch when we reach min widgets[self._bar_index] = progressbar.AnimatedMarker() self.pbar = progressbar.ProgressBar(max_value=_max, widgets=widgets) self.pbar.start()
def example26(): """ Display progress bar using tqdm. >>> example26() True """ widgets = [ progressbar.Percentage(), " ", progressbar.Bar(), " ", progressbar.ETA(), " ", progressbar.AdaptiveETA(), " ", progressbar.AdaptiveTransferSpeed(), ] pbar = progressbar.ProgressBar(widgets=widgets, max_value=500) pbar.start() for i in range(500): sleep(0.01 + (i < 100) * 0.0001 + (i > 400) * 0.009) pbar.update(i + 1) pbar.finish() return True
def show_progress(block_num, block_size, total_size): global pbar if pbar is None: if total_size > 0: prefixes = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi') power = min(int(math.log(total_size, 2) / 10), len(prefixes) - 1) scaled = float(total_size) / (2**(10 * power)) total_size_str = '{:.1f} {}B'.format(scaled, prefixes[power]) try: marker = '█' except UnicodeEncodeError: marker = '*' widgets = [ progressbar.Percentage(), ' ', progressbar.DataSize(), ' / ', total_size_str, ' ', progressbar.Bar(marker=marker), ' ', progressbar.ETA(), ' ', progressbar.AdaptiveTransferSpeed(), ] pbar = progressbar.ProgressBar(widgets=widgets, max_value=total_size) else: widgets = [ progressbar.DataSize(), ' ', progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', progressbar.Timer(), ' ', progressbar.AdaptiveTransferSpeed(), ] pbar = progressbar.ProgressBar(widgets=widgets, max_value=progressbar.UnknownLength) downloaded = block_num * block_size if downloaded < total_size: pbar.update(downloaded) else: pbar.finish() pbar = None
def train(train_times): prog = progressbar.ProgressBar( max_value=train_times * travel_data.BATCH_SIZE, widgets=[ "Train: ", progressbar.Percentage(), progressbar.Bar(), progressbar.AdaptiveETA(), "|", progressbar.AdaptiveTransferSpeed(unit="smpl") ]) for i in range(train_times): batch_x, batch_y = travel_data.feed_data() if i % 10 == 0: test_x, test_y = travel_data.feed_test_data() results = sess.run(y, feed_dict={x: test_x}) ans = test_y hr = [] for p in range(len(ans)): s_hit = 0 s_total = 0 res = { x: results[p][x] for x in range(travel_data.CITY_COUNT) } res = sorted(res.items(), key=lambda x: x[1], reverse=True)[:10] res = [item[0] for item in res] for j in range(len(ans[p])): if ans[p][j] == 1: s_total += 1 if j in res: s_hit += 1 hr.append(s_hit / s_total) hr = sum(hr) / len(hr) train_writer.add_summary( sess.run(merged, feed_dict={ x: batch_x, y_ans: batch_y, hit_rate: hr }), i) if i % 100 == 0: run_options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() sess.run(train_step, feed_dict={ x: batch_x, y_ans: batch_y }, options=run_options, run_metadata=run_metadata) train_writer.add_run_metadata(run_metadata, 'step%d' % i) else: sess.run(train_step, feed_dict={x: batch_x, y_ans: batch_y}) prog.update(i * travel_data.BATCH_SIZE) prog.finish()
async def extract_tarball(file_path: str, target_path: str, compression: str = None): mode = "r|" if compression is not None and compression in {'bz2', 'gz', 'xz'}: mode += compression else: mode += "*" _logger.info("Extracting %s to %s", file_path, target_path) mkdirs(target_path) speed = progressbar.AdaptiveTransferSpeed(samples=datetime.timedelta(seconds=5)) speed.INTERVAL = datetime.timedelta(milliseconds=500) eta = progressbar.AdaptiveETA(samples=datetime.timedelta(seconds=5)) eta.INTERVAL = datetime.timedelta(milliseconds=500) bar = progressbar.ProgressBar(widgets=[ ' ', progressbar.Timer(format='%(elapsed)s'), f' - Reading {file_path} @ ', speed, ' - ', progressbar.AnimatedMarker() ], redirect_stdout=True) with bar: bar.start() with __open_tarball(file_path, mode) as tar: for member in tar: member.name = re.sub(r'[:]', '_', member.name) if not member.isfile(): continue fileobj = tar.extractfile(member) if fileobj is None: continue rel_path = member.name.split("/") target_file = join(target_path, *rel_path) mkdirs(dirname(target_file)) async with aopen(target_file, "wb") as f: while True: chunk = fileobj.read() if not chunk: break await f.write(chunk) bar += member.size
def adaptive_eta_without_value_change(): # Testing progressbar.AdaptiveETA when the value doesn't actually change bar = progressbar.ProgressBar(widgets=[ progressbar.AdaptiveETA(), progressbar.AdaptiveTransferSpeed(), ], max_value=2, poll_interval=0.0001) bar.start() for i in range(100): bar.update(1) time.sleep(0.1) bar.finish()
def make_progress_bar(): return progressbar.ProgressBar(redirect_stdout=True, redirect_stderr=True, widgets=[ progressbar.Percentage(), progressbar.Bar(), ' (', progressbar.AdaptiveTransferSpeed(), ' ', progressbar.ETA(), ') ', ])
def test_adaptive_transfer_speed(): '''Testing (Adaptive)ETA when the value doesn't actually change''' widgets = [ progressbar.AdaptiveTransferSpeed(), ] p = progressbar.ProgressBar(max_value=2, widgets=widgets, poll_interval=0.0001) p.start() p.update(1) time.sleep(0.001) p.update(1) p.finish()
def initializee_progress_bar(max_value): if max_value is None: max_value = progressbar.UnknownLength widgets = [ progressbar.Percentage(), ' (', progressbar.DataSize(), ' of', progressbar.DataSize('max_value'), ' )', progressbar.AdaptiveTransferSpeed(), progressbar.Bar(marker='█'), progressbar.Timer(), ' ', progressbar.AdaptiveETA() ] return progressbar.DataTransferBar(max_value=max_value, widgets=widgets)
def make_progress_bar(name, size): widgets = [ '%s: ' % name[:8], progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.AdaptiveETA(), ' ', progressbar.DataSize(), ' ', progressbar.AdaptiveTransferSpeed(), ] return progressbar.ProgressBar(widgets=widgets, max_value=size)
def __init__(self, total: int = None): widgets = [ " [", progressbar.Timer(), "] ", progressbar.DataSize(), progressbar.Bar(), progressbar.AdaptiveTransferSpeed(), " (", progressbar.ETA(), ") ", ] self.bar = progressbar.ProgressBar(max_value=total, widgets=widgets) self.seen_so_far = 0
def test_etas(monkeypatch): '''Compare file transfer speed to adaptive transfer speed''' n = 10 interval = datetime.timedelta(seconds=1) widgets = [ progressbar.FileTransferSpeed(), progressbar.AdaptiveTransferSpeed(samples=n / 2), ] datas = [] # Capture the output sent towards the `_speed` method def calculate_eta(self, value, elapsed): '''Capture the widget output''' data = dict( value=value, elapsed=elapsed, ) datas.append(data) return 0, 0 monkeypatch.setattr(progressbar.FileTransferSpeed, '_speed', calculate_eta) for widget in widgets: widget.INTERVAL = interval p = progressbar.ProgressBar( max_value=n, widgets=widgets, poll_interval=interval, ) # Run the first few samples at a low speed and speed up later so we can # compare the results from both widgets for i in range(n): p.update(i) if i > n / 2: time.sleep(1) else: time.sleep(10) p.finish() for i, (a, b) in enumerate(zip(datas[::2], datas[1::2])): # Because the speed is identical initially, the results should be the # same for adaptive and regular transfer speed. Only when the speed # changes we should start see a lot of differences between the two if i < (n / 2 - 1): assert a['elapsed'] == b['elapsed'] else: assert a['elapsed'] > b['elapsed']
async def write_streaming_binary(data: AsyncIterator[bytes], target: str, chunk_size: int = 8192, full_size: Optional[int] = None): if chunk_size <= 0 or not isinstance(chunk_size, int): raise ValueError("The chunk size must be a positive integer") mkdirs(dirname(target)) speed = progressbar.AdaptiveTransferSpeed(samples=datetime.timedelta(seconds=5)) speed.INTERVAL = datetime.timedelta(milliseconds=500) if full_size is not None and full_size > 0 and isinstance(chunk_size, int): eta = progressbar.AdaptiveETA(samples=datetime.timedelta(seconds=5)) eta.INTERVAL = datetime.timedelta(milliseconds=500) widgets = [ ' ', progressbar.Timer(format='%(elapsed)s'), ' ', progressbar.Bar(left='[', right=']'), ' ', progressbar.Percentage(), ' - ', progressbar.DataSize(), '/', progressbar.DataSize('max_value'), ' @ ', speed, ' (', eta, ') ', ] bar = progressbar.ProgressBar(max_value=full_size, widgets=widgets, redirect_stdout=True) else: widgets = [ ' ', progressbar.Timer(format='%(elapsed)s'), ' - ', progressbar.DataSize(), ' @ ', speed, ' - ', progressbar.AnimatedMarker() ] bar = progressbar.ProgressBar(widgets=widgets, redirect_stdout=True, redirect_stderr=True) async with aopen(target, "wb") as f: with bar: async for chunk in data: await f.write(chunk) bar += len(chunk)
def download_progressbar(total_size): """ Create a progress bar to show in real-time a download status """ # Compute DownaloadProgressBar max value if total_size <= 0: max_val = progressbar.UnknownLength else: max_val = int(total_size / CHUNK_SIZE) # DownaloadProgressBar settings MARKER = '█' PREFIXES = ('', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')[1:] POLL_INTERVAL = 0.8 # DownaloadProgressBar spacing LEFT_SPACE = 4 PERCENTAGE_SPACE = 4 PRE_BAR_SPACE = 1 BAR_SPACE = 35 POST_BAR_SPACE = 1 DATA_SIZE_SPACE = 8 PRE_SPEED_SPACE = 1 SPEED_SPACE = 8 # Compute right spacing, and ensure that is not negative try: right_space = int(get_terminal_size()[0]) - \ LEFT_SPACE - PERCENTAGE_SPACE - PRE_BAR_SPACE - BAR_SPACE - \ POST_BAR_SPACE - DATA_SIZE_SPACE - PRE_SPEED_SPACE - SPEED_SPACE if right_space < 0: right_space = 0 except (ValueError, TypeError, ArithmeticError): right_space = 0 # Define DownaloadProgressBar skin bar_skin = ([ LEFT_SPACE * ' ', progressbar.Percentage(), PRE_BAR_SPACE * ' ', progressbar.Bar(marker=MARKER), POST_BAR_SPACE * ' ', progressbar.DataSize(prefixes=PREFIXES), PRE_SPEED_SPACE * ' ', progressbar.AdaptiveTransferSpeed(prefixes=PREFIXES), right_space * ' ' ]) # Generate DownaloadProgressBar return progressbar.ProgressBar(max_value=max_val, widgets=bar_skin, poll_interval=POLL_INTERVAL)
def sendFile(fileName): global sendQueueCounter # Path of file to send filePath = os.path.abspath(os.path.join(VIDEODIR, fileName)) # NOTE: Disabling traversal check for now. This should be implemented in # the media manager # No traversal please... #if not filePath.startswith(VIDEODIR): # log.error("You're trying to send something that isn't in your Video directory. Bailing.") # return # Figure out the file size fileSize = os.path.getsize(filePath) # Progress bar to monitor progress bar = progressbar.ProgressBar(widgets=[ ' [', progressbar.Percentage(), '] ', progressbar.Bar(), ' (', progressbar.ETA(), ', ', progressbar.AdaptiveTransferSpeed(), ') ', ], max_value=fileSize) with open(filePath, "rb") as f: data = f.read(SENDSIZE) totalRead = len(data) # So long as we're reading data, send it while data != b"": bar.update(totalRead) # TODO: Rework this "protocol". Right now, it's just going to write files, because... #YOLO sendQueue.put(priority=PRIORITY_TRANSFER, msg=dill.dumps({ 'type': 'fileTransfer', 'fileName': fileName, 'data': data })) data = f.read(SENDSIZE) totalRead += len(data) bar.finish()
def __init__(self, total): widgets = [ progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA(), ' ', progressbar.AdaptiveETA(), ' ', progressbar.AdaptiveTransferSpeed(), ] self.bar = progressbar.ProgressBar(widgets=widgets, maxval=total) self.bar.start()
def __init__(self, sc64: SC64) -> None: self.__sc64 = sc64 self.__widgets = [ "[ ", progressbar.FormatLabel("{variables.label}", new_style=True), " | ", progressbar.Bar(left="", right=""), " ", progressbar.Percentage(), " | ", progressbar.DataSize(prefixes=(" ", "Ki", "Mi")), " | ", progressbar.AdaptiveTransferSpeed(), " ]" ] self.__variables = {"label": ""} self.__bar = None
def download_file(drive, sourceid, dest, filename, filesize): request = drive.files().get_media(fileId=sourceid) dest_file = dest / filename with open(dest_file, 'wb') as f, progressbar.ProgressBar( max_value=filesize, widgets=[ f'downloading {dest_file.as_posix()} ', progressbar.AdaptiveTransferSpeed(), ' ', progressbar.Bar(), ' ', progressbar.AdaptiveETA() ]) as pbar: downloader = MediaIoBaseDownload(f, request, chunksize=1024 * 1024) done = False while not done: status, done = downloader.next_chunk(num_retries=3) if status: pbar.update(status.resumable_progress)
def test_etas(monkeypatch): '''Compare file transfer speed to adaptive transfer speed''' n = 10 interval = datetime.timedelta(seconds=1) widgets = [ progressbar.FileTransferSpeed(), progressbar.AdaptiveTransferSpeed(samples=n / 2), ] datas = [] # Capture the output sent towards the `_speed` method def calculate_eta(self, value, elapsed): '''Capture the widget output''' data = dict( value=value, elapsed=int(elapsed), ) datas.append(data) return 0, 0 monkeypatch.setattr(progressbar.FileTransferSpeed, '_speed', calculate_eta) monkeypatch.setattr(progressbar.AdaptiveTransferSpeed, '_speed', calculate_eta) for widget in widgets: widget.INTERVAL = interval p = progressbar.ProgressBar( max_value=n, widgets=widgets, poll_interval=interval, ) # Run the first few samples at a low speed and speed up later so we can # compare the results from both widgets for i in range(n): p.update(i) if i > n / 2: time.sleep(1) else: time.sleep(10) p.finish()
def test_all_widgets_large_values(): widgets = [ progressbar.Timer(), progressbar.ETA(), progressbar.AdaptiveETA(), progressbar.FileTransferSpeed(), progressbar.AdaptiveTransferSpeed(), progressbar.AnimatedMarker(), progressbar.Counter(), progressbar.Percentage(), progressbar.FormatLabel('%(value)d/%(max_value)d'), progressbar.SimpleProgress(), progressbar.Bar(), progressbar.ReverseBar(), progressbar.BouncingBar(), ] p = progressbar.ProgressBar(widgets=widgets, max_value=10**6) for i in range(0, 10**6, 10**4): time.sleep(0.001) p.update(i + 1) p.finish()
def download(url: str, path: str, logger: logging.Logger): logger.debug(f'download url is: {url}') r = requests.get(url=url, stream=True) if os.path.isdir(path): cd = r.headers['Content-Disposition'] fn = cd.removeprefix('attachment; filename=').removeprefix( '"').removesuffix('"') fn = fn.translate(dict.fromkeys([c for c in range(128, 255)], '')).replace(' ', ' ') path = os.path.join(path, fn) else: fn = os.path.basename(path) logger.info(f'save to "{fn}".') bar = progressbar.ProgressBar(redirect_stdout=True, redirect_stderr=True, widgets=[ progressbar.Percentage(), progressbar.Bar(), ' (', progressbar.AdaptiveTransferSpeed(), ' ', progressbar.ETA(), ') ', ]) with atomicwrites.atomic_write(path, mode='wb') as f: total_length = int(r.headers.get('content-length')) bar.start(total_length) readsofar = 0 for chunk in r.iter_content(chunk_size=1024): if chunk: readsofar += len(chunk) bar.update(readsofar) f.write(chunk) f.flush() bar.finish()
def get_progressbar(prefix, total_size, start_size=0): ''' 得到需要的进度条对象。 args: prefix:进度条前面的信息,string; total_size:文件总大小; start_size:文件已经下载的大小,用于断点续传; return: ProgressBar对象 ''' widgets = [ pb.Percentage(), " ", pb.Bar(">"), " ", pb.AdaptiveETA(), " ", pb.AdaptiveTransferSpeed(samples=datetime.timedelta(seconds=1)) ] if prefix is not None: widgets = [prefix, " "] + widgets bar = pb.ProgressBar(widgets=widgets, min_value=start_size, max_value=total_size) return bar
def example27(): """ Display progress bar using tqdm. >>> example27() True """ # Testing AdaptiveETA when the value doesn't actually change pbar = progressbar.ProgressBar( widgets=[ progressbar.AdaptiveETA(), progressbar.AdaptiveTransferSpeed() ], max_value=2, poll=0.0001, ) pbar.start() pbar.update(1) sleep(0.01) pbar.update(1) pbar.finish() return True
def download_mod_files(save_location, mod_files: List[ModFile]): session = requests.Session() total = 0 total_obtained = 0 for mod_file in mod_files: total += mod_file.file_length widgets = [ progressbar.widgets.Percentage(), ' of ', progressbar.DataSize('max_value'), ' @ ', progressbar.AdaptiveTransferSpeed(), ' ', progressbar.Bar(), ' ', progressbar.Timer(), ' ', progressbar.AdaptiveETA() ] # Preallocate space. for mod_file in mod_files: logger.debug(f'Preallocating space for {mod_file.file_name}') with open(Path(save_location).joinpath(mod_file.file_name), 'wb') as fd: fd.truncate(mod_file.file_length) with progressbar.ProgressBar(max_value=total, widgets=widgets).start() as bar: for mod_file in mod_files: logger.info(f'Downloading: {mod_file.file_name}') bar.update(total_obtained) r = session.get(mod_file.download_url, stream=True) #content_length = int(r.headers['content-length']) r.raise_for_status() with open(Path(save_location).joinpath(mod_file.file_name), 'wb') as fd: for chunk in r.iter_content(chunk_size=128): total_obtained = total_obtained + len(chunk) bar.update(total_obtained) fd.write(chunk)
def compute(self): print("Entered compute()") #Initial Tantrum self.logConfig("FileStatus", "Downloading") self.noc = mp.cpu_count() self.logConfig("NOC", self.noc) #Process calculations self.nop = self.sigmoid(self.fileSize) #Defining download pointers. self.distribute() #Distribute the download pointers #setting up tracker and ProgressBar self.index = 0 self.tracker = range(0, self.fileSize) self.tracker.append(self.fileSize) self.pbar = progressbar.ProgressBar( max_value=self.fileSize, widgets=[ progressbar.AdaptiveTransferSpeed(), progressbar.Timer(), progressbar.Bar(), progressbar.ETA() ])