def custom_progress_text(message): import progressbar from string import Formatter message_ = message.replace('(', '{') message_ = message_.replace(')', '}') keys = [key[1] for key in Formatter().parse(message_)] ids = {} for key in keys: if key is not None: ids[key] = float('nan') msg = progressbar.FormatCustomText(message, ids) return msg
def test_format_custom_text_widget(): widget = progressbar.FormatCustomText( 'Spam: %(spam).1f kg, eggs: %(eggs)d', dict( spam=0.25, eggs=3, ), ) bar = progressbar.ProgressBar(widgets=[ widget, ]) for i in bar(range(5)): widget.update_mapping(eggs=i * 2) assert widget.mapping['eggs'] == bar.widgets[0].mapping['eggs']
def __init__(self, stat_format=''): """Construct instance of data pipeline progress bar.""" self.statuses = StatView() self.stat_format = stat_format self.format_custom_text = progressbar.FormatCustomText( '(%(school)s) ==%(mode)s== %(stats)s', ) self.bar = progressbar.ProgressBar(redirect_stdout=True, max_value=progressbar.UnknownLength, widgets=[ ' [', ProgressBar.Timer(), '] ', self.format_custom_text, ])
def extractFile(self, filename): def on_progress(filename, position, total_size, pb): pass def get_file_progress_file_object_class(on_progress, pb): class FileProgressFileObject(tarfile.ExFileObject): def read(self, size, *args): on_progress(self.name, self.position, self.size, pb) return tarfile.ExFileObject.read(self, size, *args) return FileProgressFileObject class ProgressFileObject(io.FileIO): def __init__(self, path, pb, *args, **kwargs): self.pb = pb self._total_size = os.path.getsize(path) io.FileIO.__init__(self, path, *args, **kwargs) def read(self, size): self.pb.update(self.tell()) return io.FileIO.read(self, size) #: terms = shutil.get_terminal_size((100, 100)) # filler = 0 # if terms[0] > 100: # filler = int(terms[0]/4) widgets = [ progressbar.FormatCustomText("Extracting : {:25.25}".format(os.path.basename(filename))), " ", progressbar.Percentage(), " ", progressbar.Bar(fill=chr(9617), marker=chr(9608), left="[", right="]"), " ", progressbar.DataSize(), "/", progressbar.DataSize(variable="max_value"), # " "*filler, ] pbar = progressbar.ProgressBar(widgets=widgets, maxval=os.path.getsize(filename)) pbar.start() tarfile.TarFile.fileobject = get_file_progress_file_object_class(on_progress, pbar) tar = tarfile.open(fileobj=ProgressFileObject(filename, pbar), mode="r:*") outputPath = os.path.commonprefix(tar.getnames()) if os.path.isfile(outputPath): return outputPath tar.close() pbar.finish() else: tar.extractall() tar.close() pbar.finish() return outputPath
def get_progress_bar(total): format_custom_text = progressbar.FormatCustomText( 'Loss: %(loss).3f | Acc: %(acc).3f%% (%(c)d/%(t)d)', dict( loss=0, acc=0, c=0, t=0, ), ) prog_bar = progressbar.ProgressBar(0, total, widgets=[ progressbar.Counter(), ' of {} '.format(total), progressbar.Bar(), ' ', progressbar.ETA(), ' ', format_custom_text ]) return prog_bar, format_custom_text
def get_progressbar_widget_list(nepoch: int) -> BarWidgetsReturn: epoch_status_fmt_str: str = 'EPOCH: %(epoch_ix)d/%(nepoch)d' epoch_status = progressbar.FormatCustomText( epoch_status_fmt_str, dict(epoch_ix=0, nepoch=nepoch)) widgets_list: List[widgets.WidgetBase] = [ widgets.Percentage(), ' ', widgets.SimpleProgress(format='(%s)' % widgets.SimpleProgress.DEFAULT_FORMAT), ' ', epoch_status, ' ', widgets.Bar(), ' ', widgets.Timer(), ] return widgets_list, epoch_status
def format_custom_text(): format_custom_text = progressbar.FormatCustomText( 'Spam: %(spam).1f kg, eggs: %(eggs)d', dict( spam=0.25, eggs=3, ), ) bar = progressbar.ProgressBar(widgets=[ format_custom_text, ' :: ', progressbar.Percentage(), ]) for i in bar(range(25)): format_custom_text.update_mapping(eggs=i * 2) time.sleep(0.1)
def __init__(self): self.format_custom_text = progressbar.FormatCustomText( '(%(school)s) ==%(mode)s== ', ) self.bar = progressbar.ProgressBar(redirect_stdout=True, max_value=progressbar.UnknownLength, widgets=[ ' [', Timer(), '] ', self.format_custom_text, progressbar.Bar(), '(', progressbar.ETA(), ')', ])
def __init__(self, stat_format="", statistics=None): """Construct instance of data pipeline progress bar.""" self.statistics = statistics or StatView() self.stat_format = stat_format self.format_custom_text = progressbar.FormatCustomText( "(%(school)s) ==%(mode)s== %(stats)s", ) self.bar = progressbar.ProgressBar( redirect_stdout=True, # max_value=progressbar.UnknownLength, widgets=[ " [", Timer(), "] ", self.format_custom_text, ], )
def runner_parallel(inp): function, args, name, queue, order_idx = inp idx = int(multiprocessing.current_process().name.split("-")[1]) custom_text = progressbar.FormatCustomText( '{} - %(type_op)s: '.format(name), dict(type_op="Start")) widget_style = [ custom_text, progressbar.Percentage(), ' (', progressbar.SimpleProgress(), ')', ' ', progressbar.Bar(), ' ', progressbar.Timer(), ' ', progressbar.AdaptiveETA() ] args += [ ProgressBar(widgets=widget_style, fd=Writer((0, idx - 1), queue)), custom_text ] return (function(*args), order_idx)
def __init__(self, present="", past=None, max_value=1, vars=None, **kwargs): self.present = present self.sub_bar = None self.finished = None if past is None: past = present self.msg_bar = MessageBar( msg=present, finish_msg="%s finished in" % past) widgets = [self.msg_bar, " "] if max_value is None: widgets.append(progressbar.Timer(format="%(elapsed)s")) else: widgets.append(progressbar.ETA( format="ETA: %(eta)s", format_finished="%(elapsed)s")) if vars is not None: self.var_vals = progressbar.FormatCustomText( " (" + ", ".join("%s: %%(%s)s" % (v, v) for v in vars) + ")", {v: "---" for v in vars}) widgets.append(self.var_vals) else: self.var_vals = None def update_thread(): while not self.finished: if self.sub_bar is None or self.sub_bar.finished: self.update() time.sleep(0.001) self.thread = threading.Thread(target=update_thread) self.thread.daemon = True if max_value is None: max_value = progressbar.UnknownLength super(ProgressBar, self).__init__( poll_interval=0.1, widgets=widgets, fd=sys.stdout, max_value=max_value, **kwargs)
def _create_progress_bar(cls, n_fitting_tasks, scheduler): cls.tasks_text = progressbar.FormatCustomText( '(%(tasks)d / %(n_tasks)d)', dict(tasks=0, n_tasks=n_fitting_tasks)) cls.progress_bar = progressbar.ProgressBar( max_value=n_fitting_tasks, widgets=[progressbar.Percentage(), ' ', cls.tasks_text]) cls.fitting_task = 1 progress_bar_msg = str( ) if scheduler is 'multiprocessing' else 'Progress: ' if all([ hasattr(cls, attribute) for attribute in ['ind', 'dataset_name', 'n_datasets'] ]): progress_bar_msg = 'Current dataset: {} | Completed datasets: {}/{} | ' + progress_bar_msg cls.progress_bar.prefix = progress_bar_msg.format( cls.dataset_name, cls.ind, cls.n_datasets) else: cls.progress_bar.prefix = progress_bar_msg
def __init__(self): self.format_custom_text = progressbar.FormatCustomText( "(%(school)s) ==%(mode)s== ", ) self.bar = progressbar.ProgressBar( redirect_stdout=True, max_value=progressbar.UnknownLength, widgets=[ " [", Timer(), "] ", self.format_custom_text, progressbar.Bar(), "(", progressbar.ETA(), ")", ], )
def __init__(self, n_epochs, n_batches): self.text = pbar.FormatCustomText( 'Epoch: %(epoch).d/%(n_epochs).d, Batch: %(batch)d/%(n_batches)d', dict(epoch=0, n_epochs=n_epochs, batch=0, n_batches=n_batches), ) self.bar = pbar.ProgressBar(widgets=[ pbar.Percentage(), ' ', self.text, ' ', pbar.Bar(), ' ', pbar.Timer(), ' ', pbar.AdaptiveETA(), ' ', ], redirect_stdout=True) self.bar.start()
def download_file(self, url=None, outputFileName=None, outputPath=None, bytes=False): def fmt_size(num, suffix="B"): for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: if abs(num) < 1024.0: return "%3.1f%s%s" % (num, unit, suffix) num /= 1024.0 return "%.1f%s%s" % (num, "Yi", suffix) #: if not url: raise Exception("No URL specified.") if outputPath is None: # Default to current dir. outputPath = os.getcwd() else: if not os.path.isdir(outputPath): raise Exception( 'Specified path "{0}" does not exist'.format(outputPath)) fileName = os.path.basename(url) # Get URL filename userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0" if 'sourceforge.net' in url.lower(): userAgent = 'wget/1.18' # sourceforce <3 wget if url.lower().startswith("ftp://"): self.log("Requesting : {0}".format(url)) if outputFileName != None: fileName = outputFileName fullOutputPath = os.path.join(outputPath, fileName) urllib.request.urlretrieve(url, fullOutputPath) return fullOutputPath req = requests.get(url, stream=True, headers={"User-Agent": userAgent}) if req.status_code != 200: req.raise_for_status() if "content-disposition" in req.headers: reSponse = re.findall("filename=(.+)", req.headers["content-disposition"]) if reSponse == None: fileName = os.path.basename(url) else: fileName = reSponse[0] size = None compressed = False if "Content-Length" in req.headers: size = int(req.headers["Content-Length"]) if "Content-Encoding" in req.headers: if req.headers["Content-Encoding"] == "gzip": compressed = True self.log("Requesting : {0} - {1}".format( url, fmt_size(size) if size != None else "?")) # terms = shutil.get_terminal_size((100,100)) # filler = 0 # if terms[0] > 100: # filler = int(terms[0]/4) widgetsNoSize = [ progressbar.FormatCustomText("Downloading: {:25.25}".format( os.path.basename(fileName))), " ", progressbar.AnimatedMarker(markers='|/-\\'), " ", progressbar.DataSize() # " "*filler ] widgets = [ progressbar.FormatCustomText("Downloading: {:25.25}".format( os.path.basename(fileName))), " ", progressbar.Percentage(), " ", progressbar.Bar(fill=chr(9617), marker=chr(9608), left="[", right="]"), " ", progressbar.DataSize(), "/", progressbar.DataSize(variable="max_value"), " |", progressbar.AdaptiveTransferSpeed(), " | ", progressbar.ETA(), # " "*filler ] pbar = None if size == None: pbar = progressbar.ProgressBar(widgets=widgetsNoSize, maxval=progressbar.UnknownLength) else: pbar = progressbar.ProgressBar(widgets=widgets, maxval=size) if outputFileName != None: fileName = outputFileName fullOutputPath = os.path.join(outputPath, fileName) updateSize = 0 if isinstance(pbar.max_value, int): updateSize = pbar.max_value if pbar.max_value < 1024 else 1024 if bytes == True: output = b"" bytesrecv = 0 pbar.start() for buffer in req.iter_content(chunk_size=1024): if buffer: output += buffer if compressed: pbar.update(updateSize) else: pbar.update(bytesrecv) bytesrecv += len(buffer) pbar.finish() return output else: with open(fullOutputPath, "wb") as file: bytesrecv = 0 pbar.start() for buffer in req.iter_content(chunk_size=1024): if buffer: file.write(buffer) file.flush() if compressed: pbar.update(updateSize) else: pbar.update(bytesrecv) bytesrecv += len(buffer) pbar.finish() return fullOutputPath
def save_continue_progress(self, info, videofilename, url): dl = True path01 = os.getcwd() while_counter = 0 slice = 1024 * 1 # 讀取影片串流區塊大小 while dl == True: iterator = 0 clist = [] piece_count = 0 print('讀取資料中.........') with open(videofilename + '.mp4', 'wb') as f: try: print('start') r = requests.get(url, stream=True, timeout=15) file_size = int(r.headers['Content-Length']) size = str(file_size) piece = int(int(size) / 1024) print('piece:{}'.format(piece)) bar = progressbar.ProgressBar( maxval=file_size, widgets=[ progressbar.Percentage(), ' : ', progressbar.Counter(), ' of ', progressbar.FormatCustomText(size), ' bytes', ' [', progressbar.AdaptiveTransferSpeed(), '] ', # progressbar.ETA(), # ' [',progressbar.Timer(), '] ', progressbar.Bar(marker='■'), ]).start() for chunk in r.iter_content(chunk_size=slice): try: def gen(): yield chunk ge = gen() if len(chunk) == 1024: f.write(ge.__next__()) clist.append(len(chunk)) # f.write(chunk) iterator += slice bar.update(iterator) piece_count += 1 if int(piece_count) == int(piece): dl = False break elif int(piece_count) == int(piece): dl = False break else: print('piece_count :{}'.format(piece_count)) print(' 不到 1MB 重抓') f.write(ge.__next__()) except: print('來此走一遭的 黑人問號') except requests.exceptions.ConnectionError: print('\n 掉線拉 ') dl = True while_counter += 1 if while_counter == 10: print('Give up...') dl = False except: print('黑人問號') finally: if dl: print(piece_count) print(' ======Disconnect ======') print(' Close file ;(') os.remove(self.ori_path + '/' + 'dannyTumblr' + '/' + 'Download' + '/' + info + '/' + 'Videos') else: print(' Download Done') print(' Close file :>') # print(clist) for i in clist: if i < 1024: print(i) print('影片總:{}字串'.format(len(clist))) f.close() return
def train(model, criterion, optimizer, dataloaders, scheduler=NoneScheduler(None), epoch=100, device=torch.device('cuda:0'), l2=0.0, metrics=(mm.Loss(), mm.CIndexForSlide()), standard_metric_index=1, clip_grad=False): best_model_wts = copy.deepcopy(model.state_dict()) best_metric = 0.0 best_metric_name = metrics[standard_metric_index].__class__.__name__ + \ '_valid' history = { m.__class__.__name__ + p: [] for p in ['_train', '_valid'] for m in metrics } model.to(device) for e in range(epoch): for phase in ['train', 'valid']: if phase == 'train': scheduler.step() model.train() prefix = "Train: " else: model.eval() prefix = "Valid: " # progressbar format_custom_text = pb.FormatCustomText('Loss: %(loss).4f', dict(loss=0.)) widgets = [ prefix, " ", pb.Counter(), ' ', pb.Bar(), ' ', pb.Timer(), ' ', pb.AdaptiveETA(), ' ', format_custom_text ] iterator = pb.progressbar(dataloaders[phase], widgets=widgets) for m in metrics: m.reset() for batch_x, batch_y, (batch_ids, batch_files) in iterator: batch_x = batch_x.to(device) batch_y = batch_y.to(device) optimizer.zero_grad() with torch.set_grad_enabled(phase == 'train'): logit = model(batch_x) loss = criterion(logit, batch_y) # 只给weight加l2正则化 if l2 > 0.0: for p_n, p_v in model.named_parameters(): if p_n == 'weight': loss += l2 * p_v.norm() if phase == 'train': loss.backward() if clip_grad: nn.utils.clip_grad_norm_(model.parameters(), max_norm=1) optimizer.step() with torch.no_grad(): for m in metrics: if isinstance(m, mm.Loss): m.add(loss.cpu().item(), batch_x.size(0)) format_custom_text.update_mapping(loss=m.value()) else: m.add(logit.squeeze(), batch_y, batch_ids) for m in metrics: history[m.__class__.__name__ + '_' + phase].append(m.value()) print("Epoch: %d, Phase:%s, " % (e, phase) + ", ".join([ '%s: %.4f' % (m.__class__.__name__, history[m.__class__.__name__ + '_' + phase][-1]) for m in metrics ])) if phase == 'valid': epoch_metric = history[best_metric_name][-1] if epoch_metric > best_metric: best_metric = epoch_metric best_model_wts = copy.deepcopy(model.state_dict()) print("Best metric: %.4f" % best_metric) model.load_state_dict(best_model_wts) return model, history
def default_widgets(self): return [pb.FormatCustomText(format = self.message), pb.Percentage(**self.widget_kwargs), pb.Bar(**self.widget_kwargs), pb.AdaptiveETA(**self.widget_kwargs)]
def _cli_packages_add_category(self, category: str): Packages.logger.info(f"Getting package info from www.whdload.de ...") all_packages = self._get_package_list(category, True) dupes = set(x for x in all_packages for y in self.data if x == y.list_info) changed = set(x for x in all_packages for y in self.data if x[1] == y.list_info[1] and x[3] != y.list_info[3]) new_packages = sorted(all_packages.difference(dupes), key=lambda x: x[0]) if len(dupes) > 0: if len(dupes) == 1: package_text = "package" else: package_text = "packages" Packages.logger.info( f"Skipping {len(dupes)} duplicate {package_text}.") if len(changed) > 0: del_indexes = [ i for x in changed for i in range(len(self.data)) if x[1] == self.data[i].list_info[1] ] for i in del_indexes: del self.data[i] if len(changed) == 1: package_text = "package" else: package_text = "packages" Packages.logger.info( f"Removed {len(changed)} outdated {package_text} from WHDLoad Package data file." ) if len(new_packages) > 0: if len(new_packages) == 1: package_text = "package" else: package_text = "packages" Packages.logger.info( f"Adding {len(new_packages)} {package_text} to WHDLoad Package data file." ) if shutil.get_terminal_size().columns >= 120: bar_name_width = 35 bar_name_format = progressbar.FormatCustomText( "%(bar_name)s", {"bar_name": repeat(" ", bar_name_width)}) bar_widgets = [ progressbar.Percentage(), " ", progressbar.Bar(left="[", right="]", fill="."), " ", progressbar.Counter("%(value)5d"), f"/{len(new_packages):d}", " | ", progressbar.Timer(format="Elapsed: %(elapsed)s"), " | ", bar_name_format, " " ] else: bar_name_width = 20 bar_name_format = progressbar.FormatCustomText( "%(bar_name)s", {"bar_name": repeat(" ", bar_name_width)}) bar_widgets = [ progressbar.Percentage(), " ", progressbar.Bar(left="[", right="]", fill="."), " ", progressbar.Timer(format="%(elapsed)s"), " | ", bar_name_format, " " ] bar = progressbar.ProgressBar(widgets=bar_widgets, min_value=0, max_value=len(new_packages)) bar_name = pad(truncate("Starting ...", bar_name_width), bar_name_width, Align.Left.value) bar_name_format.update_mapping(bar_name=bar_name) bar.update(bar.min_value) for i in range(len(new_packages)): name = new_packages[i][0] bar_name = pad(truncate(f"{name}", bar_name_width), bar_name_width, Align.Left.value) bar_name_format.update_mapping(bar_name=bar_name) DataSet.logger.info(f"Getting info for '{name}'") bar.update(i + 1) self._add_package(new_packages[i]) self.save() bar_name = pad(truncate("Finished!", bar_name_width), bar_name_width, Align.Left.value) bar_name_format.update_mapping(bar_name=bar_name) DataSet.logger.info("Finished!") bar.update(bar.max_value) else: Packages.logger.info( f"No packages to add to WHDLoad Package data file.")
##################################### def custom_progress_text(message): import progressbar from string import Formatter message_ = message.replace('(', '{') message_ = message_.replace(')', '}') keys = [key[1] for key in Formatter().parse(message_)] ids = {} for key in keys: if key is not None: ids[key] = float('nan') msg = progressbar.FormatCustomText(message, ids) return msg def create_progress_bar(text=None): import progressbar if text is None: text = progressbar.FormatCustomText('') bar = progressbar.ProgressBar(widgets=[ progressbar.Percentage(), progressbar.Bar(), progressbar.AdaptiveETA(), ' ', text, ]) return bar def display_model(model):
def track(self, files=None, startfile=None, endfile=None, preprocess=True, write=True, track=True, postprocess=True): self.progress = 0 if files is None: files = ['generator'] + self.lines if hasattr( self, 'generator') else self.lines if startfile is not None and startfile in files: index = files.index(startfile) files = files[index:] if endfile is not None and endfile in files: index = files.index(endfile) files = files[:index + 1] if self.verbose: format_custom_text = progressbar.FormatCustomText( 'File: %(running)s', {'running': ''}) bar = progressbar.ProgressBar(widgets=[ format_custom_text, progressbar.Percentage(), progressbar.Bar(), progressbar.Percentage(), ], max_value=len(files)) format_custom_text.update_mapping(running=files[0] + ' ') for i in bar(list(range(len(files)))): l = files[i] self.progress = 100. * (i + 1) / len(files) if l == 'generator' and hasattr(self, 'generator'): format_custom_text.update_mapping(running='Generator ') if write: self.generator.write() if track: self.generator.run() if postprocess: self.generator.astra_to_hdf5() else: if i == (len(files) - 1): format_custom_text.update_mapping(running='Finished') else: format_custom_text.update_mapping( running=files[i + 1] + ' ') if preprocess: self.latticeObjects[l].preProcess() if write: self.latticeObjects[l].write() if track: self.latticeObjects[l].run() if postprocess: self.latticeObjects[l].postProcess() else: for i in range(len(files)): l = files[i] self.progress = 100. * (i) / len(files) if l == 'generator' and hasattr(self, 'generator'): if write: self.generator.write() self.progress = 100. * (i + 0.33) / len(files) if track: self.generator.run() self.progress = 100. * (i + 0.66) / len(files) if postprocess: self.generator.astra_to_hdf5() else: if preprocess: self.latticeObjects[l].preProcess() self.progress = 100. * (i + 0.25) / len(files) if write: self.latticeObjects[l].write() self.progress = 100. * (i + 0.5) / len(files) if track: self.latticeObjects[l].run() self.progress = 100. * (i + 0.75) / len(files) if postprocess: self.latticeObjects[l].postProcess() self.progress = 100
# fetch issue data data from database to show to html file [Images] import sqlite3 import os import sys import query_handler import urllib import urllib.request import time import progressbar format_custom_text = progressbar.FormatCustomText( ' Checking or downloading: %(current)d out of %(images)d Images', dict( images=0, current=0, ), ) global bar bar = 0 def startBar(maxval): global bar bar = progressbar.ProgressBar( widgets=[ progressbar.SimpleProgress(), format_custom_text, ' :: ', progressbar.Bar('█'), ' ', progressbar.ETA(),
# download and decrypt PDF file import urllib import urllib.request import sys import os import base64decoding import PDFdecrypt import time import progressbar global bar, downloaded bar = 0 downloaded = 0 format_custom_text = progressbar.FormatCustomText( ' Downloading: %(current)d Bytes of %(total)d Bytes', dict( total=0, current=0, ), ) def startBar(maxval): global bar bar = progressbar.ProgressBar( widgets=[ progressbar.SimpleProgress(), format_custom_text, ' :: ', progressbar.Bar('█'), ' ', progressbar.ETA(),
def train_sparse_dictionary1(self, data, sp_opt_max_iter=200, init_traindata_num=200, incr_rate=2, min_iterations=3, init_codebook_comps=None, debug=False): ''' <data> is a numpy array, holding all the features(of single kind) that are required to train the sparse dictionary, with dimensions [n_features, n_samples]. The sparse dictionary is trained with a random subset of <data>, which is increasing in each iteration with rate <incr_rate> , along with the max iterations <sp_opt_max_iter> of feature sign search algorithm. <min_iterations> is the least number of iterations of the dictionary training, after total data is processed. ''' self.sparse_dim = min(data.shape) * self.sparse_dim_rat self.flush_variables() try: import progressbar except: LOG.warning('Install module progressbar2 to get informed about the' +' feature sign search algorithm progress') pass self.initialize(data.shape[0], init_codebook_comps=init_codebook_comps) iter_count = 0 retry_count = 0 LOG.info('Training dictionary: ' + self.name) LOG.info('Minimum Epochs number after total data is processed:' + str(min_iterations)) reached_traindata_num = False reached_traindata_count = 0 computed = data.shape[1] * [None] retry = False lar_approx = False while True: LOG.info('Epoch: ' + str(iter_count)) loaded = False self.sparse_feat_list = None self.inp_feat_list = None if debug and iter_count == 0: LOG.warning('Debug is on, loading data from first FSS execution') try: with open(self.name+' debug_sparse.pkl','r') as inp: (self.codebook_comps, self.sparse_feat_list, self.are_sparsecoded_inp) = pickle.load(inp) loaded=True except (IOError, EOFError): LOG.warning('Not existent '+self.name +' debug_sparse.pkl') if not loaded: train_num = min(int(init_traindata_num * (incr_rate) ** iter_count), data.shape[1]) if train_num == data.shape[1] and not reached_traindata_num: reached_traindata_num = True LOG.info('Total data is processed') if reached_traindata_num: reached_traindata_count += 1 LOG.info('Number of samples used: ' + str(train_num)) ran = rand.sample(xrange(data.shape[1]), train_num) feat_sign_max_iter = min(1000, sp_opt_max_iter * incr_rate ** iter_count) LOG.info('Feature Sign Search maximum iterations allowed:' + str(feat_sign_max_iter)) try: format_custom_text = progressbar.FormatCustomText( 'Mean Initial Error: %(mean_init_energy).4f,'+ ' Mean Final Error: %(mean).4f ,Valid Samples Ratio: %(valid).2f', dict( mean_init_energy=0, mean=0, valid=0 ), ) pbar = progressbar.ProgressBar(max_value=train_num - 1, redirect_stdout=True, widgets=[progressbar.widgets.Percentage(), progressbar.widgets.Bar(), format_custom_text]) errors=True sum_error = 0 sum_energy = 0 except UnboundLocalError: pbar = None errors = False pass are_sparsecoded = [] if pbar is not None: iterat = pbar(enumerate(ran)) else: iterat = enumerate(ran) for count, sample_count in iterat: fin_error, valid, init_energy = self.feature_sign_search_algorithm( data[:, sample_count], max_iter=feat_sign_max_iter, ret_error=errors,training=True, starting_points=computed[sample_count]) are_sparsecoded.append(True) try: if iter_count > 0 and valid: #do not trust first iteration sparse features, before #having trained the codebooks at least once computed[sample_count] = self.sparse_feat_list[-1] except (TypeError,AttributeError): pass if valid and pbar and errors: sum_error += fin_error mean_error = sum_error/float(sum(are_sparsecoded)) sum_energy += init_energy mean_init_energy = sum_energy/float(sum(are_sparsecoded)) if pbar is not None: format_custom_text.update_mapping(mean_init_energy= mean_init_energy, mean=mean_error, valid=sum(are_sparsecoded) /float(len(are_sparsecoded))) self.initialize(data.shape[0]) self.inp_feat_list = np.transpose(np.array(self.inp_feat_list)) self.sparse_feat_list = np.array(self.sparse_feat_list).T are_sparsecoded = np.array( are_sparsecoded).astype(bool) retry = np.sum(are_sparsecoded) < 1 / 3.0 * (are_sparsecoded).size self.are_sparsecoded_inp = self.inp_feat_list[:, are_sparsecoded] if debug and iter_count==0: LOG.warning('Debug is on, saving debug_sparse.pkl') with open(self.name + ' debug_sparse.pkl','w') as out: pickle.dump((self.codebook_comps, self.sparse_feat_list, self.are_sparsecoded_inp), out) prev_error = compute_lineq_error(self.are_sparsecoded_inp, self.codebook_comps, self.sparse_feat_list) if not lar_approx: dictionary = self.conj_grad_dict_compute() curr_error = compute_lineq_error( self.are_sparsecoded_inp, dictionary, self.sparse_feat_list) LOG.info('Reconstruction Error: ' + str(curr_error)) if loaded: mean_init_energy=0 mean_error = 0 if curr_error > prev_error or mean_error>1000 or retry or lar_approx: if (prev_error > 100 or mean_error>1000 or retry or lar_approx): if retry_count == 2 or lar_approx: if iter_count != 0: iter_count = 0 lar_approx = True init_traindata_num = data.shape[1] continue LOG.warning('Training has high final error but' + ' reached maximum retries. No codebook can' + ' be produced with the fast method,'+ ' using Lagrange Dual, as input'+ ' sparsecoded data S is' +' ill-conditioned (too low' + ' rank of the STS).'+ ' Least Angle Regression Method '+ ' will be used') self.codebook_comps = DictionaryLearning( self.sparse_dim, fit_algorithm='lars', code_init=self.inp_feat_list.T).fit( self.are_sparsecoded_inp.T).components_.T curr_error = compute_lineq_error( self.are_sparsecoded_inp, self.codebook_comps, self.sparse_feat_list) LOG.info('Reconstruction Error using LARS: ' + str(curr_error)) if curr_error > 1000: LOG.info('LARS method did not converge,' + ' no codebook is produced.') self.is_trained = False self.codebook_comps = None else: break LOG.warning('Training of codebook ' + self.name + ' completed with no success,'+ ' reinitializing (Retry:' + str(retry_count + 1) + ')') self.flush_variables() self.initialize(data.shape[0]) computed = data.shape[1] * [None] retry_count += 1 iter_count = -1 reached_traindata_count = 0 reached_traindata_num = False elif (np.isclose(prev_error,curr_error,atol=0.1) and reached_traindata_num and reached_traindata_count > min_iterations): break if curr_error < 0.5 and reached_traindata_num: break if (reached_traindata_num and reached_traindata_count > min_iterations and iter_count >= 0): break iter_count += 1 self.codebook_comps = dictionary self.inp_feat_list = None self.sparse_feat_list = None self.is_trained = True
# decrypt and save PDF file from PyPDF2 import PdfFileReader, PdfFileWriter import time import progressbar global bar bar = 0 format_custom_text = progressbar.FormatCustomText( ' Decrypting: %(current)d out of %(total)d Pages', dict( total = 0, current = 0, ), ) def startBar(maxval): global bar bar = progressbar.ProgressBar( widgets=[progressbar.SimpleProgress(), format_custom_text, ' :: ', progressbar.Bar('█'), ' ', progressbar.ETA(), ' ',], max_value=maxval, ).start() def progressCheck(current,total): format_custom_text.update_mapping(current = current, total= total) bar.update(current)
# net = net.cuda() optimizer = Adam(params=net.parameters(), lr=0.001) # loss loss = nn.NLLLoss() batch_number = len(loader) num_epochs = 500 logging_step = 50 logging_text_step = 1000 widgets = [ progressbar.DynamicMessage("epoch"), ' ', 'Batch: ', progressbar.Counter(), '/', progressbar.FormatCustomText('%(total)s', {"total": batch_number}), ' ', progressbar.Bar(marker="-", left='[', right=']'), ' ', progressbar.ETA(), ' ', progressbar.DynamicMessage('loss'), ' ', progressbar.DynamicMessage("accuracy"), ] for i in xrange(num_epochs): progress = progressbar.ProgressBar(min_value=0, max_value=batch_number, initial_value=0, widgets=widgets).start()
def train(model, criterion, optimizer, dataloaders, scheduler=NoneScheduler(None), epoch=100, device=torch.device('cuda:0'), l2=0.0, metrics=(mm.Loss(), ), standard_metric_index=1, clip_grad=False, weighter_multipler=1.0): weighter = Weighter(dataloaders['train'].dataset, device, multipler=weighter_multipler) # 构建几个变量来储存最好的模型 best_model_wts = copy.deepcopy(model.state_dict()) best_metric = 0.0 best_metric_name = metrics[standard_metric_index].__class__.__name__ + \ '_valid' best_weighter = copy.deepcopy(weighter) # 储存最好模型对应的weighter # 构建dict来储存训练过程中的结果 history = { m.__class__.__name__ + p: [] for p in ['_train', '_valid'] for m in metrics } model.to(device) for e in range(epoch): for phase in ['train', 'valid']: if phase == 'train': scheduler.step() model.train() prefix = "Train: " else: model.eval() prefix = "Valid: " # progressbar format_custom_text = pb.FormatCustomText('Loss: %(loss).4f', dict(loss=0.)) widgets = [ prefix, " ", pb.Percentage(), ' ', pb.SimpleProgress(format='(%s)' % pb.SimpleProgress.DEFAULT_FORMAT), ' ', pb.Bar(), ' ', pb.Timer(), ' ', pb.AdaptiveETA(), ' ', format_custom_text ] iterator = pb.progressbar(dataloaders[phase], widgets=widgets) for m in metrics: m.reset() for batch_x, batch_y, bag_ids, inst_ids in iterator: batch_x = batch_x.to(device) batch_y = batch_y.to(device) optimizer.zero_grad() with torch.set_grad_enabled(phase == 'train'): proba = model(batch_x).squeeze() # 注意模型输出的需要时proba # 计算每个样本的权重 w = weighter(proba, batch_y, bag_ids, inst_ids) # 这个criterion不能reduction loss_es = criterion(proba, batch_y.float()) # 使用计算的权重 loss = (loss_es * w).mean() # 只给weight加l2正则化 if l2 > 0.0: for p_n, p_v in model.named_parameters(): if p_n == 'weight': loss += l2 * p_v.norm() if phase == 'train': loss.backward() if clip_grad: nn.utils.clip_grad_norm_(model.parameters(), max_norm=1) optimizer.step() with torch.no_grad(): for m in metrics: if isinstance(m, mm.Loss): m.add(loss.cpu().item(), batch_x.size(0)) format_custom_text.update_mapping(loss=m.value()) else: m.add(proba.squeeze(), batch_y, bag_ids) for m in metrics: history[m.__class__.__name__ + '_' + phase].append(m.value()) print("Epoch: %d, Phase:%s, " % (e, phase) + ", ".join([ '%s: %.4f' % (m.__class__.__name__, history[m.__class__.__name__ + '_' + phase][-1]) for m in metrics ])) if phase == 'valid': epoch_metric = history[best_metric_name][-1] if epoch_metric > best_metric: best_metric = epoch_metric best_model_wts = copy.deepcopy(model.state_dict()) best_weighter = copy.deepcopy(weighter) print("Best metric: %.4f" % best_metric) model.load_state_dict(best_model_wts) return model, history, best_weighter