def log_progress(sequence, every=None, size=None): is_iterator = False start_tic = time.time() if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = size / 200 # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{index} / ?'.format(index=index) else: progress.value = index label.value = u'{index} / {size}'.format(index=index, size=size) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index # pretty human readable time diff if False: import dateutil.relativedelta attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds'] delta = dateutil.relativedelta.relativedelta(seconds=time.time() - start_tic) elapsed = " ".join([ '%d %s' % (getattr(delta, attr), getattr(delta, attr) > 1 and attr or attr[:-1]) for attr in attrs if getattr(delta, attr) ]) + " %d usec" % ((tdiff - int(tdiff)) * 1000000) label.value = u'{index} : {elapsed}'.format(index=index or '?', elapsed=elapsed) else: # simple time in sec label.value = u'{index} : {elapsed:0.2f}s'.format( index=index or '?', elapsed=time.time() - start_tic)
def log_progress(sequence, every=None, size=None, name='Items'): """Ipywidget for displaying a progress bar. Developed by Elisa Heim Args: sequence ([type]): [description] every ([type], optional): [description]. Defaults to None. size ([type], optional): [description]. Defaults to None. name (str, optional): [description]. Defaults to 'Items'. Yields: Ipywidget: a progress bar showing current / max number of models. """ from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format(name=name, index=index) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = "{name}: {index}".format(name=name, index=str(index or '?'))
def log_progress(sequence, every=None, size=None, desc='Items'): """ Make and display a progress bar. Parameters ---------- sequence : list. Represents a elements sequence. every : int, optional, default None. Represents the steps in which the bar will be updated size : int, optional, default None. Represents the size/number elements in sequence. desc : String, optional, default 'Items'. Represents the description of the operation. """ is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) else: if every is None: raise AssertionError('Sequence is iterator, set every') if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '%s: %s / ?' % (desc, index) else: progress.value = index label.value = u'%s: %s / %s' % (desc, index, size) yield record except Exception: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = '%s: %s' % (desc, str(index or '?'))
def _log_progress(sequence, desc='Items', total=None, miniters=None): """ Make and display a progress bar. Parameters ---------- sequence : list. Represents a elements sequence. desc : String, optional, default 'Items'. Represents the description of the operation. total : int, optional, default None. Represents the total/number elements in sequence. miniters : int, optional, default None. Represents the steps in which the bar will be updated """ is_iterator = False if total is None: try: total = len(sequence) except TypeError: is_iterator = True if total is not None: if miniters is None: if total <= 200: miniters = 1 else: miniters = int(total / 200) else: if miniters is None: miniters = 1 if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=total, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % miniters == 0: if is_iterator: label.value = '%s: %s / ?' % (desc, index) else: progress.value = index label.value = u'%s: %s / %s' % (desc, index, total) yield record except Exception: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = '%s: %s' % (desc, str(index or '?'))
def progressBar(sequence, width=None, every=None, size=None, name='Items', text_color='white'): from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=0, width=width) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0, width=width) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 0): if index == 0 or index % every == 0: if is_iterator: label.value = '<font color="{color}">{name}: {index} / ?</font>'.format( color=text_color, name=name, index=index) else: progress.value = index label.value = u'<font color="{color}">{name}: {index} / {size}</font>'.format( color=text_color, name=name, index=index, size=size) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index + 1 label.value = u'<font color="{color}">{name}: {index} / {size}</font>'.format( color=text_color, name=name, index=str(index + 1 or '?'), size=size)
def log_progress(sequence, every=None, size=None, name='Items'): """ Displays a progress indicator Example def dothing(v): time.sleep(1) s = [1, 2, 3,4] for r in log_progress(s, every=1): dothing(r) From https://github.com/alexanderkuk/log-progress """ from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format(name=name, index=index) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = "{name}: {index}".format(name=name, index=str(index or '?'))
def log_progress(sequence, every=None, size=None, name='Items'): """https://github.com/alexanderkuk/log-progress Arguments: sequence {[type]} -- [description] Keyword Arguments: every {[type]} -- [description] (default: {None}) size {[type]} -- [description] (default: {None}) name {str} -- [description] (default: {'Items'}) """ from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format(name=name, index=index) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = "{name}: {index}".format(name=name, index=str(index or '?'))
def progress(sequence, every=None, size=None, name='Items'): from ipywidgets import IntProgress, HTML, VBox from IPython.display import display import time import datetime is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) start = time.time() index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format(name=name, index=index) else: progress.value = index elapsed = time.time() - start avg_iter = elapsed / index remaining = int(avg_iter * (size - index)) label.value = u'{name}: {index} / {size} [ETA: {seconds}]'.format( name=name, index=index, size=size, seconds=datetime.timedelta(seconds=remaining)) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index
def log_progress(sequence, every=None, size=None): is_iterator = False start_tic = time.time() if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = size / 200 # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{index} / ?'.format(index=index) else: progress.value = index label.value = u'{index} / {size}'.format( index=index, size=size ) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index # pretty human readable time diff if False: import dateutil.relativedelta attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds'] delta = dateutil.relativedelta.relativedelta(seconds=time.time()-start_tic) elapsed = " ".join(['%d %s' % (getattr(delta, attr), getattr(delta, attr) > 1 and attr or attr[:-1]) for attr in attrs if getattr(delta, attr)])+" %d usec"%((tdiff-int(tdiff))*1000000) label.value = u'{index} : {elapsed}'.format(index=index or '?', elapsed=elapsed) else: # simple time in sec label.value = u'{index} : {elapsed:0.2f}s'.format(index=index or '?', elapsed=time.time()-start_tic)
def _log_progress(sequence, every=1, size=None, name='Position', timeit=True): from ipywidgets import IntProgress, HTML, VBox, HBox from IPython.display import display progress_logger = logging.getLogger(__name__ + ".progress") is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True timeit = False if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) position_label = HTML() time_label = HTML() label = HBox(children=[position_label, time_label]) box = VBox(children=[label, progress]) display(box) index = 0 if timeit: t_remaining = "?" t_start = timer() try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: position_label.value = u'{}: {} / ?'.format(name, index) else: progress.value = index position_label.value = u'{}: {} / {}'.format(name, index, size) if timeit: time_label.value = u' | Remaining: {}'.format(_format_remaining(t_remaining)) progress_logger.info(position_label.value + time_label.value) yield record if timeit: t_remaining = (size - index - 1) * (timer() - t_start) / (index) except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index position_label.value = "{}: {}".format(name, str(index or '?'))
def progress_bar(sequence, every=None, size=None, name='Progress'): '''Source: https://github.com/kuk/log-progress. Makes progress bar during iterating through sequence. Example: from time import sleep for i in progress_bar([1,2,3,4],every=1): sleep(1)''' is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format( name=name, index=index ) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size ) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = "{name}: {index}".format( name=name, index=str(index or '?') )
def log_progress(sequence, every=None, size=None, name='Items') -> object: """ Shows a progress bar with labels in a Jupyter notebook (see https://github.com/kuk/log-progress). Args: sequence: The list to iterate over. Each element in the list can cause a progress bar update but the frequency depends on the every parameter. every: The frequency of the progress bar update. E.g. update progress bar after every two items. size: The number of items in the list. name: The description to show. Returns: """ is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = f'{name}: {record} / ?' else: progress.value = index label.value = f'{name}: {record} / {sequence[-1]}' yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = f'{name}: {record} / {sequence[-1]}'
def log_progress(sequence, every=None, size=None, name='Items'): from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format( name=name, index=index ) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size ) yield record except Exception: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = "{name}: {index}".format( name=name, index=str(index or '?') )
def log_progress(sequence, every=None, size=None): """ Returns a log progress bar for computations """ from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = size / 200 # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{index} / ?'.format(index=index) else: progress.value = index label.value = u'{index} / {size}'.format( index=index, size=size ) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = str(index or '?')
def log_progress(sequence, every=None, size=None): """Widget based progress bar for Jupyter (IPython Notebook) Author: Kukushkin Alexander Source: https://github.com/alexanderkuk/log-progress """ from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = size / 200 # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{index} / ?'.format(index=index) else: progress.value = index label.value = u'{index} / {size}'.format( index=index, size=size ) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = str(index or '?')
def log_progress(sequence, every=None, size=None, name='Items'): is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format( name=name, index=index ) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size ) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = "{name}: {index}".format( name=name, index=str(index or '?') )
def log_progress(sequence, every=None, size=None, name='Items'): """ Widget based progress bar for Jupyter, source: https://github.com/kuk/log-progress (requires the jupyter lab extension to be installed: $jupyter labextension install @jupyter-widgets/jupyterlab-manager) """ is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format(name=name, index=index) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = "{name}: {index}".format(name=name, index=str(index or '?'))
def progress_bar(sequence, every=None, size=None, name='Progress'): '''Source: https://github.com/kuk/log-progress. Makes progress bar during iterating through sequence. Example: from time import sleep for i in progress_bar([1,2,3,4],every=1): sleep(1)''' is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format(name=name, index=index) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = "{name}: {index}".format(name=name, index=str(index or '?'))
def log_progress(sequence, every=None, size=None, name="Items"): from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, "sequence is iterator, set every" if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = "info" else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = "{name}: {index} / ?".format(name=name, index=index) else: progress.value = index label.value = u"{name}: {index} / {size}".format( name=name, index=index, size=size) yield record except Exception: progress.bar_style = "danger" raise else: progress.bar_style = "success" progress.value = index label.value = "{name}: {index}".format(name=name, index=str(index or "?"))
def _multi_kfold_scoring(dataset, algo, L=10, k=2): """ Performs multiple scorings of the given dataset. Parameters ---------- dataset : rankeval.dataset.Dataset The dataset instance. algo : function See :func:`bias_variance`. L : int Number of iterations k : int Number of folds. Returns ------- score : numpy.ndarray A matrix num_instances x L. """ progress_bar = IntProgress(min=0, max=L, description="Computing L scores") display(progress_bar) scores = np.zeros( (dataset.n_instances, L), dtype=np.float32) for l in range(L): progress_bar.value += 1 scores[:,l] = _kfold_scoring(dataset, k, algo) progress_bar.bar_style = "success" progress_bar.close() return scores
def loading_bar_ipython(loading, data=None): from ipywidgets import IntProgress, HTML, VBox from IPython.display import display progress = IntProgress(min=0, max=100, value=0) progress.bar_style = 'info' label = HTML() box = VBox(children=[label, progress]) display(box) page_cnt = 0 shown_categories = set() while True: if data and 'pages' in data: page_cnt = len(data['pages']) if data and 'categories' in data: new_cats = set(data['categories'].keys()) for new in new_cats.difference(shown_categories): print(f"\rCategory title: {new}") shown_categories = new_cats label.value = f'Pages: {page_cnt} / ?' progress.value = page_cnt % 95 if not loading['status']: label.value = f'Pages: {page_cnt} / {page_cnt}' progress.value = 100 print(f"Scraping Done! {page_cnt} pages read") break time.sleep(0.5)
def log_progress(sequence, every=None, size=None): from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, msg in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = 'Running: {index} / ?: {msg}...'.format( index=index, msg=msg) else: progress.value = index label.value = u'Running: {index} / {size}: {msg}...'.format( index=index, size=size, msg=msg) except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = size # label.value = str(index or '?') label.value = 'Done.'
def progress_bar(sequence, every=None, size=None, name='Parameters'): from ipywidgets import IntProgress, HTML, VBox # c&p from https://github.com/kuk/log-progress is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) IPython.display.display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format(name=name, index=index) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index label.value = f"{name}: {str(index or '?')}"
def statistical_significance(datasets, model_a, model_b, metrics, n_perm=100000): """ This method computes the statistical significance of the performance difference between model_a and. Parameters ---------- datasets : list of Dataset The datasets to use for analyzing the behaviour of the model using the given metrics and models model_a : RTEnsemble The first model considered. model_b : RTEnsemble The second model considered. metrics : list of Metric The metrics to use for the analysis n_perm : int Number of permutations for the randomization test. Returns ------- stat_sig : xarray.DataArray A DataArray containing the statistical significance of the performance difference between any pair of models on the given dataset. """ progress_bar = IntProgress(min=0, max=len(datasets) * len(metrics), description="Iterating datasets and metrics") display(progress_bar) data = np.zeros(shape=(len(datasets), len(metrics), 2), dtype=np.float32) for idx_dataset, dataset in enumerate(datasets): y_pred_a = model_a.score(dataset, detailed=False) y_pred_b = model_b.score(dataset, detailed=False) for idx_metric, metric in enumerate(metrics): progress_bar.value += 1 metrics_a = metric.eval(dataset, y_pred_a)[1] metrics_b = metric.eval(dataset, y_pred_b)[1] p1, p2 = _randomization(metrics_a, metrics_b, n_perm=n_perm) data[idx_dataset][idx_metric][0] = p1 data[idx_dataset][idx_metric][1] = p2 progress_bar.bar_style = "success" progress_bar.close() performance = xr.DataArray( data, name='Statistical Significance', coords=[datasets, metrics, ['one-sided', 'two-sided']], dims=['dataset', 'metric', 'p-value']) return performance
def log_progress(sequence, name, every=None, size=None, who="reads"): from ipywidgets import IntProgress, HTML, VBox from IPython.display import display is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = size / 200 # every 0.5% else: assert every is not None, "sequence is iterator, set every" if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = "info" else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = VBox(children=[label, progress]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = name + ": " + "{index} {who} / ?".format(index=index, who=who) else: progress.value = index label.value = u"{name}: {index} / {size}".format(name=name, index=index, size=size) yield record except: progress.bar_style = "danger" raise else: progress.bar_style = "success" progress.value = index label.value = str(name + ": " + str(index) + " " + str(who) or "?")
def _kfold_scoring(dataset, k, algo): """ Scored the given datset with the given algo unsing k-fold train/test. Parameters ---------- dataset : rankeval.dataset.Dataset The dataset instance. k : int Number of folds. algo : function See :func:`bias_variance`. Returns ------- score : numpy.ndarray A vecotr of num_instances scores. """ progress_bar = IntProgress(min=0, max=k, description="Processing k folds") display(progress_bar) scores = np.zeros(dataset.n_instances, dtype=np.float32) query_sizes = dataset.get_query_sizes() # shuffle queries shuffled_qid = np.random.permutation(dataset.n_queries) chunk_query_size = int(math.ceil(dataset.n_queries/float(k))) for p in range(0, dataset.n_queries, chunk_query_size): progress_bar.value += 1 # p-th fold is used for testing test_rows = np.full(dataset.n_instances, fill_value=False, dtype=np.bool) for q in shuffled_qid[p: p + chunk_query_size]: test_rows[dataset.query_offsets[q]:dataset.query_offsets[q+1]] = True # other folds are used for training train_rows = np.logical_not(test_rows) train_q = np.full(dataset.n_queries, fill_value=True, dtype=np.bool) train_q[shuffled_qid[p: p+chunk_query_size]] = False # get algorithm predictions fold_scores = algo( dataset.X[train_rows], dataset.y[train_rows], query_sizes[train_q], dataset.X[test_rows] ) # update scores for the current fold scores[test_rows] = fold_scores progress_bar.bar_style = "success" progress_bar.close() return scores
def progressify(iterable, n): start_time = time.time() progress = IntProgress(min=0, max=n, layout=Layout(width='100%')) text = Label(layout=Layout(width='100%')) display(progress) display(text) for it in iterable: yield it progress.value += 1 elapsed_time = time.time() - start_time percent = progress.value * 100.0 / n progress.description = '%.1f%% (%s / %s)' % (percent, progress.value, n) text.value = 'elapsed %s' % datetime.timedelta(seconds=elapsed_time) progress.bar_style = 'success'
def progress_bar(job): '''Display a status bar showing how many tasks are completed''' status=get_status(job) f = IntProgress(min=0, max=status['Total'], bar_style='success') f.value = status['Total'] - status['Pending'] - status['Running'] f.description = "{:1.0f} tasks left ".format(status['Pending']+status['Running']) display(f) while f.value < status['Total']: status=get_status(job) f.value = status['Total'] - status['Pending'] - status['Running'] f.description = "{:1.0f} tasks left ".format(status['Pending']+status['Running']) if status['Failed'] > 0: f.bar_style='warning' sleep(1)
def get_market_data(self, product, start_date, end_date, interval=60, overwrite=False): """ Get market data over a range of dates Parameters: product (str) : product of data start_date (str) : date of data end_date (str) : date of data interval (int) : interval of data overwrite (bool) : overwrite cached data Returns: DataFrame : df of market data across the given time period """ start_dt = datetime(int(start_date[:4]), int(start_date[4:6]), int(start_date[6:8])) end_dt = datetime(int(end_date[:4]), int(end_date[4:6]), int(end_date[6:8])) if self.notebook_logging: start_time = time.time() delta = end_dt - start_dt total_days = delta.days + 1 print(f'Getting {product} data from {start_date} to {end_date} at {interval}s granularity') f = IntProgress(min=0, max=total_days, description = 'Progress', bar_style='info') display(f) dfs = [] # iterate and get daily data while start_dt <= end_dt: date_str = start_dt.strftime('%Y%m%d') if overwrite: dfs.append(self.get_single_day_from_api(product, date_str, interval)) else: dfs.append(self.get_single_day_market_data(product, date_str, interval)) start_dt += timedelta(days=1) if self.notebook_logging: f.value += 1 if self.notebook_logging: f.bar_style = 'success' print(f'Completed in {round(time.time() - start_time, 2)} seconds') return pd.concat(dfs, ignore_index=True)
def extract_features(model, filenames, path_to_images, display_bar=True): ''' Extracts output from model on list of filenames. Parameters ---------- model : Keras model. filenames : list List of filenames to process. path_to_images : path to folder with images Returns ------- features : ndarray Extracted features. ''' if display_bar: bar = IntProgress(value=1, min=1, max=len(filenames), step=1, description='Initializing...') display(bar) features = np.zeros((len(filenames), 4096), dtype=np.float) for i in range(len(filenames)): img_path = os.path.join(path_to_images, filenames[i]) img = image.load_img(img_path, target_size=(224, 224)) x = image.img_to_array(img) x = np.expand_dims(x, axis=0) x = preprocess_input(x) features[i] = feature_extractor.predict(x).squeeze() if display_bar: bar.value = i + 1 bar.description = '[{:>{tab}} / {}]'.format(i + 1, bar.max, tab=len(str(bar.max))) if display_bar: bar.bar_style = 'success' return features
def status_printer(_, total=None, desc=None, ncols=None): # Prepare IPython progress bar from ipywidgets import IntProgress, HTML, HBox, Layout, Label if total: pbar = IntProgress(min=0, max=total) else: # No total? Show info style bar with no progress tqdm status pbar = IntProgress(min=0, max=1) pbar.value = 1 pbar.bar_style = 'info' if desc: description = Label(desc) description_box = HBox(children=[description]) description_box.layout.min_width = '35%' description_box.layout.max_width = '35%' else: description_box = None # Prepare status text ptext = HTML() # Prepare layout inner = HBox([pbar, ptext], layout=Layout(padding='0 0 0 20px')) # Only way to place text to the right of the bar is to use a container box_layout = Layout(display='flex', width='100%') container = HBox(children=[description_box, inner] if description_box else [inner], layout=box_layout) if ncols is not None: # use default style of ipywidgets # ncols could be 100, "100px", "100%" ncols = str(ncols) # ipywidgets only accepts string if ncols[-1].isnumeric(): # if last value is digit, assume the value is digit ncols += 'px' pbar.layout.flex = '2' container.layout.width = ncols container.layout.display = 'inline-flex' container.layout.flex_flow = 'row wrap' from IPython.core.display import display display(container) # HTML encoding try: # Py3 from html import escape except ImportError: # Py2 from cgi import escape def print_status(s='', close=False, bar_style=None, desc=None): # Note: contrary to native tqdm, s='' does NOT clear bar # goal is to keep all infos if error happens so user knows # at which iteration the loop failed. # Clear previous output (really necessary?) # clear_output(wait=1) # Get current iteration value from format_meter string if total: # n = None if s: npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...' # Check that n can be found in s (else n > total) if npos >= 0: n = int(s[:npos]) # get n from string s = s[npos + 3:] # remove from string # Update bar with current n value if n is not None: pbar.value = n # Print stats if s: # never clear the bar (signal: s='') s = s.replace('||', '') # remove inesthetical pipes s = escape(s) # html escape special characters (like '?') ptext.value = s # Change bar style if bar_style: # Hack-ish way to avoid the danger bar_style being overriden by # success because the bar gets closed after the error... if not (pbar.bar_style == 'danger' and bar_style == 'success'): pbar.bar_style = bar_style # Special signal to close the bar if close and pbar.bar_style != 'danger': # hide only if no error try: container.close() except AttributeError: container.visible = False # Update description if desc: description.value = desc return print_status
def animator(progbar, count, tot_string=False, linenum=False, terminal=False, init=False, length=False, **kwargs ): """ Animates progress bar in unique position in terminal Multiple progress bars not supported in jupyter yet. """ # if ipython? welcome_message = kwargs.pop('welcome_message', '') if welcome_message: welcome_message = welcome_message.replace('Interrogating corpus ... \n', '') welcome_message = welcome_message.replace('Concordancing corpus ... \n', '') welcome_message = welcome_message.replace('\n', '<br>').replace(' ' * 17, ' ' * 17) else: welcome_message = '' if init: from traitlets import TraitError try: from ipywidgets import IntProgress, HTML, VBox from IPython.display import display progress = IntProgress(min=0, max=length, value=1) using_notebook = True progress.bar_style = 'info' label = HTML() label.font_family = 'monospace' gblabel = HTML() gblabel.font_family = 'monospace' box = VBox(children=[label, progress, gblabel]) display(box) return box except TraitError: pass except ImportError: pass if not init: try: from ipywidgets.widgets.widget_box import FlexBox if isinstance(progbar, FlexBox): label, progress, goodbye = progbar.children progress.value = count if count == length: progress.bar_style = 'success' else: label.value = '%s\nInterrogating: %s ...' % (welcome_message, tot_string) return except: pass # add startnum start_at = kwargs.get('startnum', 0) if start_at is None: start_at = 0.0 denominator = kwargs.get('denom', 1) if kwargs.get('note'): if count is None: perc_done = 0.0 else: perc_done = (count * 100.0 / float(length)) / float(denominator) kwargs['note'].progvar.set(start_at + perc_done) kwargs['root'].update() return if init: from corpkit.textprogressbar import TextProgressBar return TextProgressBar(length, dirname=tot_string) # this try is for sublime text nosetests, which don't take terminal object try: with terminal.location(0, terminal.height - (linenum + 1)): if tot_string: progbar.animate(count, tot_string) else: progbar.animate(count) # typeerror for nose except: if tot_string: progbar.animate(count, tot_string) else: progbar.animate(count)
def status_printer(_, total=None, desc=None): # Prepare IPython progress bar from ipywidgets import IntProgress, HTML, HBox, Layout, Label if total: pbar = IntProgress(min=0, max=total) else: # No total? Show info style bar with no progress tqdm status pbar = IntProgress(min=0, max=1) pbar.value = 1 pbar.bar_style = 'info' if desc: description = Label(desc) description_box = HBox(children=[description]) description_box.layout.min_width = '35%' description_box.layout.max_width = '35%' else: description_box = None # Prepare status text ptext = HTML() inner = HBox([pbar, ptext], layout=Layout(padding='0 0 0 20px')) # Only way to place text to the right of the bar is to use a container box_layout = Layout(display='flex', width='100%') container = HBox( children=[description_box, inner] if description_box else [inner], layout=box_layout) from IPython.core.display import display display(container) # HTML encoding try: # Py3 from html import escape except ImportError: # Py2 from cgi import escape def print_status(s='', close=False, bar_style=None, desc=None): # Note: contrary to native tqdm, s='' does NOT clear bar # goal is to keep all infos if error happens so user knows # at which iteration the loop failed. # Clear previous output (really necessary?) # clear_output(wait=1) # Get current iteration value from format_meter string if total: # n = None if s: npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...' # Check that n can be found in s (else n > total) if npos >= 0: n = int(s[:npos]) # get n from string s = s[npos + 3:] # remove from string # Update bar with current n value if n is not None: pbar.value = n # Print stats if s: # never clear the bar (signal: s='') s = s.replace('||', '') # remove inesthetical pipes s = escape(s) # html escape special characters (like '?') ptext.value = s # Change bar style if bar_style: # Hack-ish way to avoid the danger bar_style being overriden by # success because the bar gets closed after the error... if not (pbar.bar_style == 'danger' and bar_style == 'success'): pbar.bar_style = bar_style # Special signal to close the bar if close and pbar.bar_style != 'danger': # hide only if no error try: container.close() except AttributeError: container.visible = False # Update description if desc: #nonlocal description description.value = desc return print_status
def log_progress(sequence, every=None, size=None, start_time=None, name='Items'): """ Function to provide a progress bar. --- This function is useful only when this program is run as a Jupyter notebook. In order to use this, run `jupyter nbextension enable --py --sys-prefix widgetsnbextension` prior to launching the notebook server. """ from ipywidgets import IntProgress, HTML, VBox, HBox from IPython.display import display, Javascript from IPython.display import HTML as html import datetime import uuid image_id = "loading_image_{}".format(str(uuid.uuid4())) is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if start_time is None: start_time = datetime.datetime.now() if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) label = HTML() loading_image = HTML("<img id='{}' src='images/loading.gif' width=50 align=right alt='Loading Image.' />".format(image_id)) box = VBox(children=[label, HBox([progress, loading_image])]) display(box) #display(loading_image) index = 0 try: for index, record in enumerate(sequence, 1): if start_time is not None: time_elapsed = datetime.datetime.now() - start_time else: time_elapsed = datetime.datetime.now() if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ? | Time Elapsed: {time_elapsed}'.format( name=name, index=index, time_elapsed=time_elapsed ) else: progress.value = index if index == size: progress.value = index-1 label.value = u'{name}: {index} / {size} | Time Elapsed: {time_elapsed}'.format( name=name, index=index, size=size, time_elapsed=time_elapsed ) yield record except: progress.bar_style = 'danger' hide_command = """document.getElementById('{}').style.display='none';""".format(image_id) display(Javascript(hide_command)) else: progress.bar_style = 'success' hide_command = """document.getElementById('{}').style.display='none';""".format(image_id) display(Javascript(hide_command)) if start_time is not None: time_elapsed = datetime.datetime.now() - start_time else: time_elapsed = datetime.datetime.now() progress.value = index label.value = "{name}: {index} | Total Time: {time_elapsed}".format( name=name, index=str(index or '?'), time_elapsed=time_elapsed )
def log_progress(sequence, every=None, size=None, name='Items', progress_out=widgets.Output()): from ipywidgets import IntProgress, HTML, HBox from IPython.display import display, clear_output global progress # global progress_out with progress_out: clear_output(wait=True) is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.layout={'width':'214px', 'max_width':'214px'} progress.bar_style = 'info' else: progress = IntProgress(min=0, max=size, value=0) progress.layout={'width':'214px', 'max_width':'214px'} label = HTML() box = HBox(children=[progress, label]) display(box) index = 0 try: for index, record in enumerate(sequence, 1): if index == 1 or index % every == 0: if is_iterator: label.value = '{name}: {index} / ?'.format( name=name, index=index ) else: progress.value = index label.value = u'{name}: {index} / {size}'.format( name=name, index=index, size=size ) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' progress.value = index if index == 0: label.value = "{name}: {index}".format( name=name, index=str(index or '?') ) else: label.value = 'Finished reading database'.format( name=name, index=str(index or '?') )
def fiber_analysisD(mypath,savepath, CO_lower,CO_upper,angleInc,radStep): plt.ioff() ## Upload Files ## Upload Files onlyfiles = [ f for f in listdir(mypath) if isfile(join(mypath,f)) ] images = numpy.empty(len(onlyfiles), dtype=object) names = numpy.empty(len(onlyfiles), dtype=object) for n in range(0, len(onlyfiles)): images[n] = cv2.imread( join(mypath,onlyfiles[n]) ) names[n]= (onlyfiles[n]) img = numpy.empty(len(images), dtype=object) for n in range(0,len(images)): img[n]= np.invert(images[n]) ## Set up empty Matrices (can clean this up later ) ell_data_test = numpy.empty(len(img),dtype=object) Mirtheta1RadFinal_test =numpy.empty(len(img),dtype=object) MirnormPower_test = numpy.empty(len(img),dtype=object) normPower_test = numpy.empty(len(img),dtype=object) theta1RadFinal_test = numpy.empty(len(img),dtype=object) A_test = numpy.empty(len(img),dtype=object) centroid_test = numpy.empty(len(img),dtype=object) t_new_test = numpy.empty(len(img),dtype=object) im = numpy.empty(len(img), dtype=object) PF = numpy.empty(len(img),dtype=object) PS = numpy.empty(len(img),dtype=object) theta1Rad_test = numpy.empty(len(img),dtype=object) PowerY_test = numpy.empty(len(img),dtype=object) Mirtheta1RadFinal1_plot= [] MirnormPower_plot=[] th_plot=[] r_line_plot=[] normPower1_plot=[] num=0 index=num plt.rcParams.update({'figure.max_open_warning': 0}) ## avoid warning output mpl.rcParams['figure.max_open_warning'] = 0 maxiter = 100 ## begin analysis print('Number of Images:',len(img)) max_count=100 tot_sum_progress = len(img) PROGRESSBAR = IntProgress(min=0, max=max_count,description='Progress') PROGRESSBAR.bar_style='info' display(PROGRESSBAR) # display the bar for n in range(0,len(img)): PROGRESSBAR.value = int(n/tot_sum_progress*100) # signal to increment the progress bar im=img[n][:,:,1] # select first image M= im.shape[0] #get image shape N1 = im.shape[1] n1 = np.round(N1/ 2) - 1 freq = np.arange(-n1, n1 + 1, 1) x, y = freq, freq ## Fourier Transform fft_result = np.fft.fft2(im) Fshift = np.fft.fftshift(fft_result) ## adjust to power spectrum to remove imaginary components magnitude_spectrum= 20*np.log(np.abs(Fshift)) Pabs = np.abs(Fshift) ** 2 M= im.shape[0] N1=im.shape[1] # shift in terms of image because power spectrum is the mirroR of lines so # mirroring back in terms of image would give right allignment PabsFlip1 = np.rot90(Pabs) PabsFlip = np.flipud(PabsFlip1) while (PabsFlip.shape[0]>x.shape[0]): PabsFlip = np.delete(PabsFlip, (0), axis=0) while (PabsFlip.shape[1]>y.shape[0]): PabsFlip = np.delete(PabsFlip, (0), axis=1) while (PabsFlip.shape[0]<x.shape[0]): PabsFlip = np.delete(x, (0), axis=0) while (PabsFlip.shape[1]<y.shape[0]): PabsFlip = np.delete(y, (0), axis=0) #PabsFlip = np.delete(PabsFlip, (0), axis=0) #PabsFlip = np.delete(PabsFlip, (0), axis=1) logScale = plt.figure(frameon=False, figsize=(10, 10)); ax = plt.Axes(logScale, [0., 0., 1., 1.]); ax.set_axis_off(); logScale.add_axes(ax); plt.axis('off'); #plt.imshow(log(PabsFlip), cmap='gray', aspect='auto') plt.savefig(savepath+'FFT '+names[n]); #plt.show() plt.close(); # Set up polar coordinates prior to summing the spectrum theta1Rad = np.linspace(0.0, 2 * math.pi, num=int(360/angleInc)) theta1Rad_test[n]=theta1Rad f1 = np.round_(N1 / (2 * CO_lower)) f2 = np.round_(N1 / (2 * CO_upper)) rho1 = np.linspace(f1, f2, num=int((f2 - f1)/radStep)) # frequency band PowerX = np.zeros((theta1Rad.size, theta1Rad.size)) PowerY = np.zeros((theta1Rad.size)) PF[n] = PabsFlip # Interpolate using a Spine PowerSpline = scipy.interpolate.RectBivariateSpline(y=y, x=x, z=PabsFlip) PS[n] = PowerSpline n_dx = 0.001 for p in range(0, theta1Rad.size): # converting theta1Rad and rho1 to cartesian coordinates xfinal = rho1 * math.cos(theta1Rad[p]) yfinal = rho1 * math.sin(theta1Rad[p]) # Evaluate spin on path px = PowerSpline.ev(yfinal, xfinal) PowerY[p] = np.sum(px) #print(PowerY) PowerY_test[n]=PowerY theta1Rad = theta1Rad_test[n] PowerY = PowerY_test[n] #print(PowerY) # Only use the data in the first two quadrants (Spectrum is symmetric) num = len(theta1Rad) PowerYFinal = PowerY[0:num // 2] theta1RadFinal = theta1Rad[0:num // 2] theta1RadFinal_test[n]= theta1RadFinal power_area = np.trapz(PowerYFinal, theta1RadFinal) normPower = PowerYFinal / power_area normPower_test[n]=normPower # Combine data into [XY] to fit to an ellipse Mirtheta1RadFinal1 = np.concatenate([theta1RadFinal.T, (theta1RadFinal + np.pi).T]) #print('round1',Mirtheta1RadFinal1) MirnormPower = np.concatenate([normPower.T, normPower.T]) theta = Mirtheta1RadFinal1 #print('theta',theta) radius = MirnormPower #print('radius',MirnormPower) ydata = radius * np.cos(theta) xdata = radius * np.sin(theta) ell_data = np.vstack([xdata, ydata]) ell_data = ell_data.T ell_data_test[n] = ell_data Mirtheta1RadFinal_test[n] = Mirtheta1RadFinal1 MirnormPower_test[n]= MirnormPower # Python fitting function, see EllipseDirectFit XY = ell_data_test[n] centroid = np.mean(XY, axis=0) e1 = (XY[:, 0] - centroid[0]) ** 2 e2 = (XY[:, 0] - centroid[0]) * ((XY[:, 1] - centroid[1])) e3 = (XY[:, 1] - centroid[1]) ** 2 D1 = np.vstack(([e1, e2, e3])).T f1 = XY[:, 0] - centroid[0] f2 = XY[:, 1] - centroid[1] f3 = np.ones((1, XY.shape[0])) D2 = vstack(([f1, f2, f3])).T S1 = dot(D1.T, D1) S2 = dot(D1.T, D2) S3 = dot(D2.T, D2) T = dot(-linalg.inv(S3), S2.T) M = S1 + (dot(S2, T)) M = np.vstack(([M[2, :] / 2, - M[1, :], M[0, :] / 2])) _eval, evec = linalg.eig(M) cond = (4 * evec[0, :] * evec[2, :]) - (evec[1, :] ** 2) A1 = evec[:, nonzero(cond > 0)[0]] A = np.vstack((A1, dot(T, A1))) A3 = A[3] - 2 * A[0] * centroid[0] - A[1] * centroid[1] A4 = A[4] - 2 * A[2] * centroid[1] - A[1] * centroid[0] A5 = A[5] + A[0] * centroid[0] ** 2 + A[2] * centroid[1] ** 2 + A[1] * centroid[0] * centroid[1] - A[3] * centroid[ 0] - A[4] * centroid[1] A[3] = A3 A[4] = A4 A[5] = A5 A = A / linalg.norm(A) if (abs(A[1]) < (1 * 10 ^ (-15))): if ((A[0]) <= (A[2])): # Ellipse is horizontal angle = 0; major = sqrt(1 / A[0]) minor = sqrt(1 / A[2]) else: angle = np.pi/2; major = sqrt(1 / A[2]) minor = sqrt(1 / A[0]) else: R = ((A[2] - A[0]) / A[1]) tg = R - sqrt((R * R) + 1) angle = math.atan(tg) P = (2 * tg) / (1 + (tg * tg)) if ((A[0] > 0 and A[1] > 0 and A[2] > 0)): if (angle < (-pi / 4)): angle = angle + np.pi else: angle = angle elif ((A[1] / P <= (-A[1] / P))): if (angle < 0): angle = angle + np.pi / 2 else: angle = angle - np.pi / 2 elif (A[0] < 0 and A[1] < 0 and A[2] < 0): if (angle < 0): angle = angle + np.pi else: angle = angle - np.pi else: # Switch if (angle < 0): angle = angle + np.pi / 2 else: angle = angle - np.pi / 2 t_New = (angle) #angle t_new_test[n] = abs((angle* 180/np.pi)-90)-(2*angle* 180/np.pi) Mirtheta1RadFinal1= Mirtheta1RadFinal_test[n] #print('round2',Mirtheta1RadFinal1) MirnormPower = MirnormPower_test[n] #print('mirnormpower',MirnormPower) t = t_new_test[n] r_line = np.arange(0, max(MirnormPower) + .5, .5) th = np.zeros(len(r_line)) for i in range(0, len(r_line)): th[i] = t th = np.concatenate([th, (th + 180)]) r_line = np.concatenate([r_line, r_line]) ## Polar Graph graphs = ["#71C4BE","#BBDBA0","#CE7E56","#BC5C81","#84355A"]; palette = seaborn.set_palette(graphs); seaborn.set_context("poster",font_scale=1.5); seaborn.set_style('ticks') csfont = {'fontname':'Times New Roman', 'size':'14', } ticksfont = {'fontname':'Times New Roman'} plt.figure(frameon=False, figsize=(15, 10)); plt.polar(Mirtheta1RadFinal1, abs(MirnormPower), 'o',ms=5); plt.suptitle(abs(t)-180, fontsize=20); plt.polar(th * pi/180, r_line,color="#CE7E56",linewidth=4); plt.savefig(savepath+'polarpositive'+names[n]); if (max(MirnormPower)<2): inc = 0.5 elif (max(MirnormPower)<5): inc = 1 elif max(MirnormPower)<20: inc = 5 else: inc = 10 plt.yticks(np.arange(inc, max(MirnormPower), inc));#, **ticksfont) #plt.show() t_final_rad = t*pi/180 def fitted_func(thetas, c): int_value, int_err = scipy.integrate.quadrature(func=lambda x: exp(c * cos(x)), a=0.0, b=np.pi) return ((np.pi * (1.0 / np.pi * (int_value))) ** - 1) * \ np.exp(c * np.cos(2 * (thetas - t_final_rad))) c0 = 15 xdata = theta1RadFinal_test[n] theta1RadFinal = theta1RadFinal_test[n] ydata= normPower_test[n] normPower = normPower_test[n] kappa, kappa_pcov = scipy.optimize.curve_fit(f=fitted_func, p0=(c0,), xdata=theta1RadFinal_test[n], ydata=normPower_test[n]) # Shift data for plotting purposes diff = abs(theta1RadFinal - (t * pi / 180)) #print(diff,'diff') #centerLoc=diff.find(diff==min(diff)) centerLoc = diff[np.argmin(diff)]#find(diff== min(diff)) #print(centerLoc,'centerLoc') num_angles = len(theta1RadFinal) shift = ((round(num_angles / 2) - (num_angles - centerLoc))) #print(shift,'shift') normPower1 = np.roll(normPower, -int(shift)) normPower1 theta1RadFinal1 = np.roll(theta1RadFinal, -int(shift)) if (shift > 0): s = num_angles - shift s= np.asscalar(s) for k in range(s, num_angles): theta1RadFinal1[k] = np.pi + theta1RadFinal1[k] elif (shift < 0): for k in range(0,90): theta1RadFinal1[k] = -np.pi + theta1RadFinal1[k] ## Normalized Distribution Graph cartDist = plt.figure(figsize=(10, 10)); # Creates a figure containing cartesian distribution. h2 = plt.bar((theta1RadFinal1 * 180 / pi), normPower1,color="#dedede"); plt.xticks(np.arange(-360, 360, 45,), **ticksfont); plt.xlim([-95, 95]); p_act = fitted_func(theta1RadFinal1, kappa); #print(p_act) a = np.vstack([[p_act],[(theta1RadFinal1 * 180 / pi)]]); a= a.T b = a[a[:,1].argsort()]#np.sort(a,axis=1) h3, = plt.plot(b[:,1],b[:,0], linewidth=4,color="#CE7E56"); plt.xlabel('Angle (°)', **csfont); plt.ylabel('Normalized Intensity', **csfont); #print(max(normPower)) if (max(normPower)<2): inc = 0.5 elif (max(normPower)<5): inc = 1 elif (max(normPower)<20): inc = 5 else: inc = 10 plt.yticks(np.arange(0, max(normPower1) + .3, inc), **ticksfont); plt.ylim([0, max(normPower1) + .3]); normPower1_plot.append(normPower1); plt.savefig(savepath+'Normdist '+names[n]); #plt.show() #cartDist.savefig(dir + 'cartDist_' + number.__str__(), bbox_inches='tight') plt.close(); t_final=t slope, intercept, rValue, pValue, stderr = scipy.stats.linregress(p_act, normPower1) #print(rValue**2) # Rounding results for Title of Figure krnd = math.ceil(k * 1000) / 1000 thrnd = abs(math.ceil(t_final * 1000) / 1000)-180 krnd = math.trunc(krnd * 100) / 100 thrnd = math.trunc(thrnd * 100) / 100 a = 32.02 b= -12.43 c = 47.06 d = -0.9185 e = 19.43 f = -0.07693 x = kappa[0] sig = math.exp(b*x) + c*math.exp(d*x) + e*exp(f*x) #print("name",names[n],"kappa",kappa,"sigma",sig,"Rsq",rValue**2,"mu",t_final) sum_NP = sum(normPower1) #normPower2 = normPower1/sum(normPower1) #sum_NP2 = sum(normPower2) #print(sum_NP2) ## Combined plot figure palette = seaborn.set_palette(graphs); seaborn.set_context("notebook",font_scale=1.5); seaborn.set_style('ticks') csfont = {'fontname':'Times New Roman', 'size':'18', } ticksfont = {'fontname':'Times New Roman'} fig = plt.figure(frameon=False,figsize=(20,12)); fig.subplots_adjust(top=0.95); fig.suptitle('%s '%(names[n]), fontsize=34, fontweight='bold'); grid = plt.GridSpec(2, 5,hspace=0,wspace=0,width_ratios=[1, 1, 0.3, 1,1]); cart_dist = plt.subplot(grid[1, 0:2]); cart_dist.set_title('Von Mises Distribution (k)',fontsize=20); h2 = cart_dist.bar((theta1RadFinal1 * 180 / pi), normPower1,color="#dedede"); cart_dist.set_xlim([-92, 92]); cart_dist.set_xlabel('Angle (°)',fontsize=18); cart_dist.set_ylim([0, max(normPower1) + .3]); cart_dist.set_ylabel('Normalized Intensity',fontsize=18); p_act = fitted_func(theta1RadFinal1, kappa) a = np.vstack([[p_act],[(theta1RadFinal1 * 180 / pi)]]) a= a.T b = a[a[:,1].argsort()]#np.sort(a,axis=1) h3, = cart_dist.plot(b[:,1],b[:,0], linewidth=4,color="#CE7E56"); if (max(normPower)<2): inc = 0.5 elif (max(normPower)<5): inc = 1 elif (max(normPower)<20): inc = 5 else: inc = 10 polar_dist = plt.subplot(grid[0:2,3:5],projection='polar'); polar_dist.plot(Mirtheta1RadFinal1, abs(MirnormPower), 'o'); polar_dist.plot(th * pi/180, r_line,color="#CE7E56",linewidth=4); polar_dist.set_title('Ellipse Fit (u)',fontsize=20).set_position([.5, 1.08]); fig.text(0.75, 0.05, ' \n Kappa: %s '%(krnd)+' mu: %s \n'%(round((abs(t_final)-180),2))+' R^2: %s '%(round((rValue**2),2))+' sigma: %s \n'%(round(sig,2)),fontsize=20, fontweight='bold', bbox={'facecolor': '#BC5C81', 'alpha': 0.5, 'pad': 0.4}); #polar_dist.text(np.radians(-110),1.95, ' \n Kappa: %s '%(krnd)+' mu: %s \n'%(round((abs(t_final)-180),2))+' R^2: %s '%(round((rValue**2),2))+' sigma: %s \n'%(round(sig,2)),fontsize=20, fontweight='bold', bbox={'facecolor': '#BC5C81', 'alpha': 0.5, 'pad': 0.4}) orig_fig = plt.subplot(grid[0, 0]); orig_fig.set_title('Original Image',fontsize=20).set_position([.5, 1.0]); orig_fig.axis('off'); fft_fig = plt.subplot(grid[0, 1]); fft_fig.set_title('Fast Fourier Transform',fontsize=20).set_position([.5, 1.0]); fft_fig.set_axis_off(); fft_fig.axis('off'); #fft_fig.imshow(log(PabsFlip), cmap='gray') #orig_fig.imshow(images[n],cmap=plt.cm.viridis ) #plt.tight_layout() plt.savefig(savepath+'Grouped'+names[n]); #print(n,'n') #plt.show() polar_dist_2 = plt.figure(figsize=(10, 10)); #print('linebeforeindex') #print(index,'currentindex') #Mirtheta1RadFinal1_plot[index]=Mirtheta1RadFinal1; #MirnormPower_plot[index]=MirnormPower; #th_plot[index] = th * pi/180 #print(th_plot[index],'thplotindex') #r_line_plot[index]=r_line Mirtheta1RadFinal1_plot.append(Mirtheta1RadFinal1); MirnormPower_plot.append(MirnormPower); th_plot.append(th * pi/180) r_line_plot.append(r_line) polar_dist.plot(Mirtheta1RadFinal1, MirnormPower); polar_dist.plot(th * pi/180, r_line,linewidth=4); polar_dist.set_title('Ellipse Fit (u)',fontsize=20).set_position([.5, 1.08]); plt.close('all') #index=index+1 #print(index,'Index') ## export of cartesian coordinate data arr = numpy.array(MirnormPower_plot) arr2 = numpy.array(Mirtheta1RadFinal1_plot) arr3 = numpy.array(normPower1_plot) arr4 = numpy.array(theta1RadFinal1*180/pi) a = numpy.transpose(arr3) b = numpy.transpose(arr4) b[b> 100] = 0 b[b<-100] = 0 export_data=np.hstack(((b[:,None]),a)) numpy.savetxt(savepath+"data_export.csv", export_data, delimiter=",") return export_data
def status_printer(file, total=None, desc=None): """ Manage the printing of an IPython/Jupyter Notebook progress bar widget. """ # Fallback to text bar if there's no total # DEPRECATED: replaced with an 'info' style bar # if not total: # return super(tqdm_notebook, tqdm_notebook).status_printer(file) fp = file if not getattr(fp, 'flush', False): # pragma: no cover fp.flush = lambda: None # Prepare IPython progress bar if total: pbar = IntProgress(min=0, max=total) else: # No total? Show info style bar with no progress tqdm status pbar = IntProgress(min=0, max=1) pbar.value = 1 pbar.bar_style = 'info' if desc: pbar.description = desc # Prepare status text ptext = HTML() # Only way to place text to the right of the bar is to use a container container = HBox(children=[pbar, ptext]) display(container) def print_status(s='', close=False, bar_style=None): # Note: contrary to native tqdm, s='' does NOT clear bar # goal is to keep all infos if error happens so user knows # at which iteration the loop failed. # Clear previous output (really necessary?) # clear_output(wait=1) # Get current iteration value from format_meter string if total: n = None if s: npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...' # Check that n can be found in s (else n > total) if npos >= 0: n = int(s[:npos]) # get n from string s = s[npos + 3:] # remove from string # Update bar with current n value if n is not None: pbar.value = n # Print stats if s: # never clear the bar (signal: s='') s = s.replace('||', '') # remove inesthetical pipes s = escape(s) # html escape special characters (like '?') ptext.value = s # Change bar style if bar_style: # Hack-ish way to avoid the danger bar_style being overriden by # success because the bar gets closed after the error... if not (pbar.bar_style == 'danger' and bar_style == 'success'): pbar.bar_style = bar_style # Special signal to close the bar if close and pbar.bar_style != 'danger': # hide only if no error container.visible = False return print_status
def status_printer(file, total=None, desc=None): """ Manage the printing of an IPython/Jupyter Notebook progress bar widget. """ # Fallback to text bar if there's no total # DEPRECATED: replaced with an 'info' style bar # if not total: # return super(tqdm_notebook, tqdm_notebook).status_printer(file) fp = file if not getattr(fp, 'flush', False): # pragma: no cover fp.flush = lambda: None # Prepare IPython progress bar if total: pbar = IntProgress(min=0, max=total) else: # No total? Show info style bar with no progress tqdm status pbar = IntProgress(min=0, max=1) pbar.value = 1 pbar.bar_style = 'info' if desc: pbar.description = desc # Prepare status text ptext = HTML() # Only way to place text to the right of the bar is to use a container container = HBox(children=[pbar, ptext]) display(container) def print_status(s='', close=False, bar_style=None): # Note: contrary to native tqdm, s='' does NOT clear bar # goal is to keep all infos if error happens so user knows # at which iteration the loop failed. # Clear previous output (really necessary?) # clear_output(wait=1) # Get current iteration value from format_meter string if total: n = None if s: npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...' # Check that n can be found in s (else n > total) if npos >= 0: n = int(s[:npos]) # get n from string s = s[npos + 3:] # remove from string # Update bar with current n value if n is not None: pbar.value = n # Print stats if s: # never clear the bar (signal: s='') s = s.replace('||', '') # remove inesthetical pipes s = escape(s) # html escape special characters (like '?') ptext.value = s # Change bar style if bar_style: # Hack-ish way to avoid the danger bar_style being overriden by # success because the bar gets closed after the error... if not (pbar.bar_style == 'danger' and bar_style == 'success'): pbar.bar_style = bar_style # Special signal to close the bar if close and pbar.bar_style != 'danger': # hide only if no error container.visible = False return print_status
def log_progress(sequence, every=None, size=None, with_time=False, name=''): from ipywidgets import IntProgress, HTML, HBox from IPython.display import display from datetime import datetime def format_time(time_value): return str(time_value).split(".")[0] is_iterator = False if size is None: try: size = len(sequence) except TypeError: is_iterator = True if size is not None: if every is None: if size <= 200: every = 1 else: every = int(size / 200) # every 0.5% else: assert every is not None, 'sequence is iterator, set every' if is_iterator: progress = IntProgress(min=0, max=1, value=1) progress.bar_style = 'info' with_time = False else: progress = IntProgress(min=0, max=size, value=0) label = HTML() box = HBox(children=[label, progress]) display(box) time_started = datetime.now() index = 0 try: for index, record in enumerate(sequence, 0): if index % every == 0: if is_iterator: label.value = '{name} {index} / ?'.format(name=name, index=index) else: if with_time: time_elapsed = datetime.now() - time_started time_left = size / index * time_elapsed - time_elapsed time_status = "| ETA: {}".format( format_time(time_left)) else: time_status = "" progress.value = index label.value = u'{name}: {index} / {size} {time}'.format( name=name, index=index, size=size, time=time_status) yield record except: progress.bar_style = 'danger' raise else: progress.bar_style = 'success' index += 1 progress.value = index label.value = "{name} {index} / {size} | Time elapsed {time_elapsed}".format( name=name, index=str(index or '?'), size=size, time_elapsed=format_time(time_started - datetime.now()), )