def log_progress(sequence, every=None, size=None):
is_iterator = False
start_tic = time.time()
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(index=index,
size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
# pretty human readable time diff
if False:
import dateutil.relativedelta
attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']
delta = dateutil.relativedelta.relativedelta(seconds=time.time() -
start_tic)
elapsed = " ".join([
'%d %s' % (getattr(delta, attr),
getattr(delta, attr) > 1 and attr or attr[:-1])
for attr in attrs if getattr(delta, attr)
]) + " %d usec" % ((tdiff - int(tdiff)) * 1000000)
label.value = u'{index} : {elapsed}'.format(index=index or '?',
elapsed=elapsed)
else:
# simple time in sec
label.value = u'{index} : {elapsed:0.2f}s'.format(
index=index or '?', elapsed=time.time() - start_tic)
def log_progress(sequence, every=None, size=None, name='Items'):
"""Ipywidget for displaying a progress bar. Developed by Elisa Heim
Args:
sequence ([type]): [description]
every ([type], optional): [description]. Defaults to None.
size ([type], optional): [description]. Defaults to None.
name (str, optional): [description]. Defaults to 'Items'.
Yields:
Ipywidget: a progress bar showing current / max number of models.
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def log_progress(sequence, every=None, size=None, desc='Items'):
"""
Make and display a progress bar.
Parameters
----------
sequence : list.
Represents a elements sequence.
every : int, optional, default None.
Represents the steps in which the bar will be updated
size : int, optional, default None.
Represents the size/number elements in sequence.
desc : String, optional, default 'Items'.
Represents the description of the operation.
"""
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200)
else:
if every is None:
raise AssertionError('Sequence is iterator, set every')
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '%s: %s / ?' % (desc, index)
else:
progress.value = index
label.value = u'%s: %s / %s' % (desc, index, size)
yield record
except Exception:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = '%s: %s' % (desc, str(index or '?'))
def _log_progress(sequence, desc='Items', total=None, miniters=None):
"""
Make and display a progress bar.
Parameters
----------
sequence : list.
Represents a elements sequence.
desc : String, optional, default 'Items'.
Represents the description of the operation.
total : int, optional, default None.
Represents the total/number elements in sequence.
miniters : int, optional, default None.
Represents the steps in which the bar will be updated
"""
is_iterator = False
if total is None:
try:
total = len(sequence)
except TypeError:
is_iterator = True
if total is not None:
if miniters is None:
if total <= 200:
miniters = 1
else:
miniters = int(total / 200)
else:
if miniters is None:
miniters = 1
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=total, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % miniters == 0:
if is_iterator:
label.value = '%s: %s / ?' % (desc, index)
else:
progress.value = index
label.value = u'%s: %s / %s' % (desc, index, total)
yield record
except Exception:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = '%s: %s' % (desc, str(index or '?'))
def progressBar(sequence,
width=None,
every=None,
size=None,
name='Items',
text_color='white'):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=0, width=width)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0, width=width)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 0):
if index == 0 or index % every == 0:
if is_iterator:
label.value = '<font color="{color}">{name}: {index} / ?</font>'.format(
color=text_color, name=name, index=index)
else:
progress.value = index
label.value = u'<font color="{color}">{name}: {index} / {size}</font>'.format(
color=text_color, name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index + 1
label.value = u'<font color="{color}">{name}: {index} / {size}</font>'.format(
color=text_color,
name=name,
index=str(index + 1 or '?'),
size=size)
def log_progress(sequence, every=None, size=None, name='Items'):
"""
Displays a progress indicator
Example
def dothing(v):
time.sleep(1)
s = [1, 2, 3,4]
for r in log_progress(s, every=1):
dothing(r)
From https://github.com/alexanderkuk/log-progress
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def log_progress(sequence, every=None, size=None, name='Items'):
"""https://github.com/alexanderkuk/log-progress
Arguments:
sequence {[type]} -- [description]
Keyword Arguments:
every {[type]} -- [description] (default: {None})
size {[type]} -- [description] (default: {None})
name {str} -- [description] (default: {'Items'})
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def progress(sequence, every=None, size=None, name='Items'):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
import time
import datetime
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
start = time.time()
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
elapsed = time.time() - start
avg_iter = elapsed / index
remaining = int(avg_iter * (size - index))
label.value = u'{name}: {index} / {size} [ETA: {seconds}]'.format(
name=name,
index=index,
size=size,
seconds=datetime.timedelta(seconds=remaining))
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
def log_progress(sequence, every=None, size=None):
is_iterator = False
start_tic = time.time()
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
# pretty human readable time diff
if False:
import dateutil.relativedelta
attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']
delta = dateutil.relativedelta.relativedelta(seconds=time.time()-start_tic)
elapsed = " ".join(['%d %s' % (getattr(delta, attr),
getattr(delta, attr) > 1 and attr or attr[:-1]) for attr in attrs if
getattr(delta, attr)])+" %d usec"%((tdiff-int(tdiff))*1000000)
label.value = u'{index} : {elapsed}'.format(index=index or '?', elapsed=elapsed)
else:
# simple time in sec
label.value = u'{index} : {elapsed:0.2f}s'.format(index=index or '?', elapsed=time.time()-start_tic)
def _log_progress(sequence, every=1, size=None, name='Position', timeit=True):
from ipywidgets import IntProgress, HTML, VBox, HBox
from IPython.display import display
progress_logger = logging.getLogger(__name__ + ".progress")
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
timeit = False
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
position_label = HTML()
time_label = HTML()
label = HBox(children=[position_label, time_label])
box = VBox(children=[label, progress])
display(box)
index = 0
if timeit:
t_remaining = "?"
t_start = timer()
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
position_label.value = u'{}: {} / ?'.format(name, index)
else:
progress.value = index
position_label.value = u'{}: {} / {}'.format(name, index, size)
if timeit:
time_label.value = u' | Remaining: {}'.format(_format_remaining(t_remaining))
progress_logger.info(position_label.value + time_label.value)
yield record
if timeit:
t_remaining = (size - index - 1) * (timer() - t_start) / (index)
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
position_label.value = "{}: {}".format(name, str(index or '?'))
def progress_bar(sequence, every=None, size=None, name='Progress'):
'''Source: https://github.com/kuk/log-progress. Makes progress bar during iterating through sequence. Example:
from time import sleep
for i in progress_bar([1,2,3,4],every=1):
sleep(1)'''
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
def log_progress(sequence, every=None, size=None, name='Items') -> object:
"""
Shows a progress bar with labels in a Jupyter notebook (see https://github.com/kuk/log-progress).
Args:
sequence: The list to iterate over. Each element in the list can cause a progress bar update but the frequency depends on the every parameter.
every: The frequency of the progress bar update. E.g. update progress bar after every two items.
size: The number of items in the list.
name: The description to show.
Returns:
"""
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = f'{name}: {record} / ?'
else:
progress.value = index
label.value = f'{name}: {record} / {sequence[-1]}'
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = f'{name}: {record} / {sequence[-1]}'
def log_progress(sequence, every=None, size=None, name='Items'):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
yield record
except Exception:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
def log_progress(sequence, every=None, size=None):
"""
Returns a log progress bar for computations
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = str(index or '?')
def log_progress(sequence, every=None, size=None):
"""Widget based progress bar for Jupyter (IPython Notebook)
Author: Kukushkin Alexander
Source: https://github.com/alexanderkuk/log-progress
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = str(index or '?')
def log_progress(sequence, every=None, size=None, name='Items'):
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
def log_progress(sequence, every=None, size=None, name='Items'):
"""
Widget based progress bar for Jupyter, source: https://github.com/kuk/log-progress
(requires the jupyter lab extension to be installed: $jupyter labextension install @jupyter-widgets/jupyterlab-manager)
"""
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def progress_bar(sequence, every=None, size=None, name='Progress'):
'''Source: https://github.com/kuk/log-progress. Makes progress bar during iterating through sequence. Example:
from time import sleep
for i in progress_bar([1,2,3,4],every=1):
sleep(1)'''
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def log_progress(sequence, every=None, size=None, name="Items"):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, "sequence is iterator, set every"
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = "info"
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = "{name}: {index} / ?".format(name=name,
index=index)
else:
progress.value = index
label.value = u"{name}: {index} / {size}".format(
name=name, index=index, size=size)
yield record
except Exception:
progress.bar_style = "danger"
raise
else:
progress.bar_style = "success"
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or "?"))
def _multi_kfold_scoring(dataset, algo, L=10, k=2):
"""
Performs multiple scorings of the given dataset.
Parameters
----------
dataset : rankeval.dataset.Dataset
The dataset instance.
algo : function
See :func:`bias_variance`.
L : int
Number of iterations
k : int
Number of folds.
Returns
-------
score : numpy.ndarray
A matrix num_instances x L.
"""
progress_bar = IntProgress(min=0, max=L, description="Computing L scores")
display(progress_bar)
scores = np.zeros( (dataset.n_instances, L), dtype=np.float32)
for l in range(L):
progress_bar.value += 1
scores[:,l] = _kfold_scoring(dataset, k, algo)
progress_bar.bar_style = "success"
progress_bar.close()
return scores
def loading_bar_ipython(loading, data=None):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
progress = IntProgress(min=0, max=100, value=0)
progress.bar_style = 'info'
label = HTML()
box = VBox(children=[label, progress])
display(box)
page_cnt = 0
shown_categories = set()
while True:
if data and 'pages' in data:
page_cnt = len(data['pages'])
if data and 'categories' in data:
new_cats = set(data['categories'].keys())
for new in new_cats.difference(shown_categories):
print(f"\rCategory title: {new}")
shown_categories = new_cats
label.value = f'Pages: {page_cnt} / ?'
progress.value = page_cnt % 95
if not loading['status']:
label.value = f'Pages: {page_cnt} / {page_cnt}'
progress.value = 100
print(f"Scraping Done! {page_cnt} pages read")
break
time.sleep(0.5)
def log_progress(sequence, every=None, size=None):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, msg in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = 'Running: {index} / ?: {msg}...'.format(
index=index, msg=msg)
else:
progress.value = index
label.value = u'Running: {index} / {size}: {msg}...'.format(
index=index, size=size, msg=msg)
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = size
# label.value = str(index or '?')
label.value = 'Done.'
def progress_bar(sequence, every=None, size=None, name='Parameters'):
from ipywidgets import IntProgress, HTML, VBox
# c&p from https://github.com/kuk/log-progress
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
IPython.display.display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = f"{name}: {str(index or '?')}"
def statistical_significance(datasets,
model_a,
model_b,
metrics,
n_perm=100000):
"""
This method computes the statistical significance of the performance difference between model_a and.
Parameters
----------
datasets : list of Dataset
The datasets to use for analyzing the behaviour of the model using the given metrics and models
model_a : RTEnsemble
The first model considered.
model_b : RTEnsemble
The second model considered.
metrics : list of Metric
The metrics to use for the analysis
n_perm : int
Number of permutations for the randomization test.
Returns
-------
stat_sig : xarray.DataArray
A DataArray containing the statistical significance of the performance difference
between any pair of models on the given dataset.
"""
progress_bar = IntProgress(min=0,
max=len(datasets) * len(metrics),
description="Iterating datasets and metrics")
display(progress_bar)
data = np.zeros(shape=(len(datasets), len(metrics), 2), dtype=np.float32)
for idx_dataset, dataset in enumerate(datasets):
y_pred_a = model_a.score(dataset, detailed=False)
y_pred_b = model_b.score(dataset, detailed=False)
for idx_metric, metric in enumerate(metrics):
progress_bar.value += 1
metrics_a = metric.eval(dataset, y_pred_a)[1]
metrics_b = metric.eval(dataset, y_pred_b)[1]
p1, p2 = _randomization(metrics_a, metrics_b, n_perm=n_perm)
data[idx_dataset][idx_metric][0] = p1
data[idx_dataset][idx_metric][1] = p2
progress_bar.bar_style = "success"
progress_bar.close()
performance = xr.DataArray(
data,
name='Statistical Significance',
coords=[datasets, metrics, ['one-sided', 'two-sided']],
dims=['dataset', 'metric', 'p-value'])
return performance
def log_progress(sequence, name, every=None, size=None, who="reads"):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, "sequence is iterator, set every"
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = "info"
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = name + ": " + "{index} {who} / ?".format(index=index, who=who)
else:
progress.value = index
label.value = u"{name}: {index} / {size}".format(name=name, index=index, size=size)
yield record
except:
progress.bar_style = "danger"
raise
else:
progress.bar_style = "success"
progress.value = index
label.value = str(name + ": " + str(index) + " " + str(who) or "?")
def _kfold_scoring(dataset, k, algo):
"""
Scored the given datset with the given algo unsing k-fold train/test.
Parameters
----------
dataset : rankeval.dataset.Dataset
The dataset instance.
k : int
Number of folds.
algo : function
See :func:`bias_variance`.
Returns
-------
score : numpy.ndarray
A vecotr of num_instances scores.
"""
progress_bar = IntProgress(min=0, max=k, description="Processing k folds")
display(progress_bar)
scores = np.zeros(dataset.n_instances, dtype=np.float32)
query_sizes = dataset.get_query_sizes()
# shuffle queries
shuffled_qid = np.random.permutation(dataset.n_queries)
chunk_query_size = int(math.ceil(dataset.n_queries/float(k)))
for p in range(0, dataset.n_queries, chunk_query_size):
progress_bar.value += 1
# p-th fold is used for testing
test_rows = np.full(dataset.n_instances,
fill_value=False,
dtype=np.bool)
for q in shuffled_qid[p: p + chunk_query_size]:
test_rows[dataset.query_offsets[q]:dataset.query_offsets[q+1]] = True
# other folds are used for training
train_rows = np.logical_not(test_rows)
train_q = np.full(dataset.n_queries,
fill_value=True,
dtype=np.bool)
train_q[shuffled_qid[p: p+chunk_query_size]] = False
# get algorithm predictions
fold_scores = algo(
dataset.X[train_rows],
dataset.y[train_rows],
query_sizes[train_q],
dataset.X[test_rows]
)
# update scores for the current fold
scores[test_rows] = fold_scores
progress_bar.bar_style = "success"
progress_bar.close()
return scores
def progressify(iterable, n):
start_time = time.time()
progress = IntProgress(min=0, max=n, layout=Layout(width='100%'))
text = Label(layout=Layout(width='100%'))
display(progress)
display(text)
for it in iterable:
yield it
progress.value += 1
elapsed_time = time.time() - start_time
percent = progress.value * 100.0 / n
progress.description = '%.1f%% (%s / %s)' % (percent, progress.value,
n)
text.value = 'elapsed %s' % datetime.timedelta(seconds=elapsed_time)
progress.bar_style = 'success'
def progress_bar(job):
'''Display a status bar showing how many tasks are completed'''
status=get_status(job)
f = IntProgress(min=0, max=status['Total'], bar_style='success')
f.value = status['Total'] - status['Pending'] - status['Running']
f.description = "{:1.0f} tasks left ".format(status['Pending']+status['Running'])
display(f)
while f.value < status['Total']:
status=get_status(job)
f.value = status['Total'] - status['Pending'] - status['Running']
f.description = "{:1.0f} tasks left ".format(status['Pending']+status['Running'])
if status['Failed'] > 0:
f.bar_style='warning'
sleep(1)
def get_market_data(self, product, start_date, end_date, interval=60, overwrite=False):
"""
Get market data over a range of dates
Parameters:
product (str) : product of data
start_date (str) : date of data
end_date (str) : date of data
interval (int) : interval of data
overwrite (bool) : overwrite cached data
Returns:
DataFrame : df of market data across the given time period
"""
start_dt = datetime(int(start_date[:4]), int(start_date[4:6]), int(start_date[6:8]))
end_dt = datetime(int(end_date[:4]), int(end_date[4:6]), int(end_date[6:8]))
if self.notebook_logging:
start_time = time.time()
delta = end_dt - start_dt
total_days = delta.days + 1
print(f'Getting {product} data from {start_date} to {end_date} at {interval}s granularity')
f = IntProgress(min=0, max=total_days, description = 'Progress', bar_style='info')
display(f)
dfs = []
# iterate and get daily data
while start_dt <= end_dt:
date_str = start_dt.strftime('%Y%m%d')
if overwrite:
dfs.append(self.get_single_day_from_api(product, date_str, interval))
else:
dfs.append(self.get_single_day_market_data(product, date_str, interval))
start_dt += timedelta(days=1)
if self.notebook_logging:
f.value += 1
if self.notebook_logging:
f.bar_style = 'success'
print(f'Completed in {round(time.time() - start_time, 2)} seconds')
return pd.concat(dfs, ignore_index=True)
def extract_features(model, filenames, path_to_images, display_bar=True):
'''
Extracts output from model on list of filenames.
Parameters
----------
model : Keras model.
filenames : list
List of filenames to process.
path_to_images : path to folder with images
Returns
-------
features : ndarray
Extracted features.
'''
if display_bar:
bar = IntProgress(value=1,
min=1,
max=len(filenames),
step=1,
description='Initializing...')
display(bar)
features = np.zeros((len(filenames), 4096), dtype=np.float)
for i in range(len(filenames)):
img_path = os.path.join(path_to_images, filenames[i])
img = image.load_img(img_path, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
features[i] = feature_extractor.predict(x).squeeze()
if display_bar:
bar.value = i + 1
bar.description = '[{:>{tab}} / {}]'.format(i + 1,
bar.max,
tab=len(str(bar.max)))
if display_bar:
bar.bar_style = 'success'
return features
def status_printer(_, total=None, desc=None, ncols=None):
# Prepare IPython progress bar
from ipywidgets import IntProgress, HTML, HBox, Layout, Label
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
description = Label(desc)
description_box = HBox(children=[description])
description_box.layout.min_width = '35%'
description_box.layout.max_width = '35%'
else:
description_box = None
# Prepare status text
ptext = HTML()
# Prepare layout
inner = HBox([pbar, ptext],
layout=Layout(padding='0 0 0 20px'))
# Only way to place text to the right of the bar is to use a container
box_layout = Layout(display='flex',
width='100%')
container = HBox(children=[description_box, inner] if description_box else [inner],
layout=box_layout)
if ncols is not None: # use default style of ipywidgets
# ncols could be 100, "100px", "100%"
ncols = str(ncols) # ipywidgets only accepts string
if ncols[-1].isnumeric():
# if last value is digit, assume the value is digit
ncols += 'px'
pbar.layout.flex = '2'
container.layout.width = ncols
container.layout.display = 'inline-flex'
container.layout.flex_flow = 'row wrap'
from IPython.core.display import display
display(container)
# HTML encoding
try: # Py3
from html import escape
except ImportError: # Py2
from cgi import escape
def print_status(s='', close=False, bar_style=None, desc=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
# n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
try:
container.close()
except AttributeError:
container.visible = False
# Update description
if desc:
description.value = desc
return print_status
def animator(progbar,
count,
tot_string=False,
linenum=False,
terminal=False,
init=False,
length=False,
**kwargs
):
"""
Animates progress bar in unique position in terminal
Multiple progress bars not supported in jupyter yet.
"""
# if ipython?
welcome_message = kwargs.pop('welcome_message', '')
if welcome_message:
welcome_message = welcome_message.replace('Interrogating corpus ... \n', '')
welcome_message = welcome_message.replace('Concordancing corpus ... \n', '')
welcome_message = welcome_message.replace('\n', '<br>').replace(' ' * 17, ' ' * 17)
else:
welcome_message = ''
if init:
from traitlets import TraitError
try:
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
progress = IntProgress(min=0, max=length, value=1)
using_notebook = True
progress.bar_style = 'info'
label = HTML()
label.font_family = 'monospace'
gblabel = HTML()
gblabel.font_family = 'monospace'
box = VBox(children=[label, progress, gblabel])
display(box)
return box
except TraitError:
pass
except ImportError:
pass
if not init:
try:
from ipywidgets.widgets.widget_box import FlexBox
if isinstance(progbar, FlexBox):
label, progress, goodbye = progbar.children
progress.value = count
if count == length:
progress.bar_style = 'success'
else:
label.value = '%s\nInterrogating: %s ...' % (welcome_message, tot_string)
return
except:
pass
# add startnum
start_at = kwargs.get('startnum', 0)
if start_at is None:
start_at = 0.0
denominator = kwargs.get('denom', 1)
if kwargs.get('note'):
if count is None:
perc_done = 0.0
else:
perc_done = (count * 100.0 / float(length)) / float(denominator)
kwargs['note'].progvar.set(start_at + perc_done)
kwargs['root'].update()
return
if init:
from corpkit.textprogressbar import TextProgressBar
return TextProgressBar(length, dirname=tot_string)
# this try is for sublime text nosetests, which don't take terminal object
try:
with terminal.location(0, terminal.height - (linenum + 1)):
if tot_string:
progbar.animate(count, tot_string)
else:
progbar.animate(count)
# typeerror for nose
except:
if tot_string:
progbar.animate(count, tot_string)
else:
progbar.animate(count)
def status_printer(_, total=None, desc=None):
# Prepare IPython progress bar
from ipywidgets import IntProgress, HTML, HBox, Layout, Label
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
description = Label(desc)
description_box = HBox(children=[description])
description_box.layout.min_width = '35%'
description_box.layout.max_width = '35%'
else:
description_box = None
# Prepare status text
ptext = HTML()
inner = HBox([pbar, ptext], layout=Layout(padding='0 0 0 20px'))
# Only way to place text to the right of the bar is to use a container
box_layout = Layout(display='flex', width='100%')
container = HBox(
children=[description_box, inner] if description_box else [inner],
layout=box_layout)
from IPython.core.display import display
display(container)
# HTML encoding
try: # Py3
from html import escape
except ImportError: # Py2
from cgi import escape
def print_status(s='', close=False, bar_style=None, desc=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
# n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
try:
container.close()
except AttributeError:
container.visible = False
# Update description
if desc:
#nonlocal description
description.value = desc
return print_status
def log_progress(sequence, every=None, size=None, start_time=None, name='Items'):
"""
Function to provide a progress bar.
---
This function is useful only when this program is run
as a Jupyter notebook.
In order to use this, run
`jupyter nbextension enable --py --sys-prefix widgetsnbextension`
prior to launching the notebook server.
"""
from ipywidgets import IntProgress, HTML, VBox, HBox
from IPython.display import display, Javascript
from IPython.display import HTML as html
import datetime
import uuid
image_id = "loading_image_{}".format(str(uuid.uuid4()))
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if start_time is None:
start_time = datetime.datetime.now()
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
loading_image = HTML("<img id='{}' src='images/loading.gif' width=50 align=right alt='Loading Image.' />".format(image_id))
box = VBox(children=[label, HBox([progress, loading_image])])
display(box)
#display(loading_image)
index = 0
try:
for index, record in enumerate(sequence, 1):
if start_time is not None:
time_elapsed = datetime.datetime.now() - start_time
else:
time_elapsed = datetime.datetime.now()
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ? | Time Elapsed: {time_elapsed}'.format(
name=name,
index=index,
time_elapsed=time_elapsed
)
else:
progress.value = index
if index == size:
progress.value = index-1
label.value = u'{name}: {index} / {size} | Time Elapsed: {time_elapsed}'.format(
name=name,
index=index,
size=size,
time_elapsed=time_elapsed
)
yield record
except:
progress.bar_style = 'danger'
hide_command = """document.getElementById('{}').style.display='none';""".format(image_id)
display(Javascript(hide_command))
else:
progress.bar_style = 'success'
hide_command = """document.getElementById('{}').style.display='none';""".format(image_id)
display(Javascript(hide_command))
if start_time is not None:
time_elapsed = datetime.datetime.now() - start_time
else:
time_elapsed = datetime.datetime.now()
progress.value = index
label.value = "{name}: {index} | Total Time: {time_elapsed}".format(
name=name,
index=str(index or '?'),
time_elapsed=time_elapsed
)
def log_progress(sequence, every=None, size=None, name='Items', progress_out=widgets.Output()):
from ipywidgets import IntProgress, HTML, HBox
from IPython.display import display, clear_output
global progress
# global progress_out
with progress_out:
clear_output(wait=True)
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.layout={'width':'214px', 'max_width':'214px'}
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
progress.layout={'width':'214px', 'max_width':'214px'}
label = HTML()
box = HBox(children=[progress, label])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
if index == 0:
label.value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
else:
label.value = 'Finished reading database'.format(
name=name,
index=str(index or '?')
)
def fiber_analysisD(mypath,savepath, CO_lower,CO_upper,angleInc,radStep):
plt.ioff()
## Upload Files
## Upload Files
onlyfiles = [ f for f in listdir(mypath) if isfile(join(mypath,f)) ]
images = numpy.empty(len(onlyfiles), dtype=object)
names = numpy.empty(len(onlyfiles), dtype=object)
for n in range(0, len(onlyfiles)):
images[n] = cv2.imread( join(mypath,onlyfiles[n]) )
names[n]= (onlyfiles[n])
img = numpy.empty(len(images), dtype=object)
for n in range(0,len(images)):
img[n]= np.invert(images[n])
## Set up empty Matrices (can clean this up later )
ell_data_test = numpy.empty(len(img),dtype=object)
Mirtheta1RadFinal_test =numpy.empty(len(img),dtype=object)
MirnormPower_test = numpy.empty(len(img),dtype=object)
normPower_test = numpy.empty(len(img),dtype=object)
theta1RadFinal_test = numpy.empty(len(img),dtype=object)
A_test = numpy.empty(len(img),dtype=object)
centroid_test = numpy.empty(len(img),dtype=object)
t_new_test = numpy.empty(len(img),dtype=object)
im = numpy.empty(len(img), dtype=object)
PF = numpy.empty(len(img),dtype=object)
PS = numpy.empty(len(img),dtype=object)
theta1Rad_test = numpy.empty(len(img),dtype=object)
PowerY_test = numpy.empty(len(img),dtype=object)
Mirtheta1RadFinal1_plot= []
MirnormPower_plot=[]
th_plot=[]
r_line_plot=[]
normPower1_plot=[]
num=0
index=num
plt.rcParams.update({'figure.max_open_warning': 0}) ## avoid warning output
mpl.rcParams['figure.max_open_warning'] = 0
maxiter = 100
## begin analysis
print('Number of Images:',len(img))
max_count=100
tot_sum_progress = len(img)
PROGRESSBAR = IntProgress(min=0, max=max_count,description='Progress')
PROGRESSBAR.bar_style='info'
display(PROGRESSBAR) # display the bar
for n in range(0,len(img)):
PROGRESSBAR.value = int(n/tot_sum_progress*100) # signal to increment the progress bar
im=img[n][:,:,1] # select first image
M= im.shape[0] #get image shape
N1 = im.shape[1]
n1 = np.round(N1/ 2) - 1
freq = np.arange(-n1, n1 + 1, 1)
x, y = freq, freq
## Fourier Transform
fft_result = np.fft.fft2(im)
Fshift = np.fft.fftshift(fft_result)
## adjust to power spectrum to remove imaginary components
magnitude_spectrum= 20*np.log(np.abs(Fshift))
Pabs = np.abs(Fshift) ** 2
M= im.shape[0]
N1=im.shape[1]
# shift in terms of image because power spectrum is the mirroR of lines so
# mirroring back in terms of image would give right allignment
PabsFlip1 = np.rot90(Pabs)
PabsFlip = np.flipud(PabsFlip1)
while (PabsFlip.shape[0]>x.shape[0]):
PabsFlip = np.delete(PabsFlip, (0), axis=0)
while (PabsFlip.shape[1]>y.shape[0]):
PabsFlip = np.delete(PabsFlip, (0), axis=1)
while (PabsFlip.shape[0]<x.shape[0]):
PabsFlip = np.delete(x, (0), axis=0)
while (PabsFlip.shape[1]<y.shape[0]):
PabsFlip = np.delete(y, (0), axis=0)
#PabsFlip = np.delete(PabsFlip, (0), axis=0)
#PabsFlip = np.delete(PabsFlip, (0), axis=1)
logScale = plt.figure(frameon=False, figsize=(10, 10));
ax = plt.Axes(logScale, [0., 0., 1., 1.]);
ax.set_axis_off();
logScale.add_axes(ax);
plt.axis('off');
#plt.imshow(log(PabsFlip), cmap='gray', aspect='auto')
plt.savefig(savepath+'FFT '+names[n]);
#plt.show()
plt.close();
# Set up polar coordinates prior to summing the spectrum
theta1Rad = np.linspace(0.0, 2 * math.pi, num=int(360/angleInc))
theta1Rad_test[n]=theta1Rad
f1 = np.round_(N1 / (2 * CO_lower))
f2 = np.round_(N1 / (2 * CO_upper))
rho1 = np.linspace(f1, f2, num=int((f2 - f1)/radStep)) # frequency band
PowerX = np.zeros((theta1Rad.size, theta1Rad.size))
PowerY = np.zeros((theta1Rad.size))
PF[n] = PabsFlip
# Interpolate using a Spine
PowerSpline = scipy.interpolate.RectBivariateSpline(y=y, x=x, z=PabsFlip)
PS[n] = PowerSpline
n_dx = 0.001
for p in range(0, theta1Rad.size):
# converting theta1Rad and rho1 to cartesian coordinates
xfinal = rho1 * math.cos(theta1Rad[p])
yfinal = rho1 * math.sin(theta1Rad[p])
# Evaluate spin on path
px = PowerSpline.ev(yfinal, xfinal)
PowerY[p] = np.sum(px)
#print(PowerY)
PowerY_test[n]=PowerY
theta1Rad = theta1Rad_test[n]
PowerY = PowerY_test[n]
#print(PowerY)
# Only use the data in the first two quadrants (Spectrum is symmetric)
num = len(theta1Rad)
PowerYFinal = PowerY[0:num // 2]
theta1RadFinal = theta1Rad[0:num // 2]
theta1RadFinal_test[n]= theta1RadFinal
power_area = np.trapz(PowerYFinal, theta1RadFinal)
normPower = PowerYFinal / power_area
normPower_test[n]=normPower
# Combine data into [XY] to fit to an ellipse
Mirtheta1RadFinal1 = np.concatenate([theta1RadFinal.T, (theta1RadFinal + np.pi).T])
#print('round1',Mirtheta1RadFinal1)
MirnormPower = np.concatenate([normPower.T, normPower.T])
theta = Mirtheta1RadFinal1
#print('theta',theta)
radius = MirnormPower
#print('radius',MirnormPower)
ydata = radius * np.cos(theta)
xdata = radius * np.sin(theta)
ell_data = np.vstack([xdata, ydata])
ell_data = ell_data.T
ell_data_test[n] = ell_data
Mirtheta1RadFinal_test[n] = Mirtheta1RadFinal1
MirnormPower_test[n]= MirnormPower
# Python fitting function, see EllipseDirectFit
XY = ell_data_test[n]
centroid = np.mean(XY, axis=0)
e1 = (XY[:, 0] - centroid[0]) ** 2
e2 = (XY[:, 0] - centroid[0]) * ((XY[:, 1] - centroid[1]))
e3 = (XY[:, 1] - centroid[1]) ** 2
D1 = np.vstack(([e1, e2, e3])).T
f1 = XY[:, 0] - centroid[0]
f2 = XY[:, 1] - centroid[1]
f3 = np.ones((1, XY.shape[0]))
D2 = vstack(([f1, f2, f3])).T
S1 = dot(D1.T, D1)
S2 = dot(D1.T, D2)
S3 = dot(D2.T, D2)
T = dot(-linalg.inv(S3), S2.T)
M = S1 + (dot(S2, T))
M = np.vstack(([M[2, :] / 2, - M[1, :], M[0, :] / 2]))
_eval, evec = linalg.eig(M)
cond = (4 * evec[0, :] * evec[2, :]) - (evec[1, :] ** 2)
A1 = evec[:, nonzero(cond > 0)[0]]
A = np.vstack((A1, dot(T, A1)))
A3 = A[3] - 2 * A[0] * centroid[0] - A[1] * centroid[1]
A4 = A[4] - 2 * A[2] * centroid[1] - A[1] * centroid[0]
A5 = A[5] + A[0] * centroid[0] ** 2 + A[2] * centroid[1] ** 2 + A[1] * centroid[0] * centroid[1] - A[3] * centroid[
0] - A[4] * centroid[1]
A[3] = A3
A[4] = A4
A[5] = A5
A = A / linalg.norm(A)
if (abs(A[1]) < (1 * 10 ^ (-15))):
if ((A[0]) <= (A[2])):
# Ellipse is horizontal
angle = 0;
major = sqrt(1 / A[0])
minor = sqrt(1 / A[2])
else:
angle = np.pi/2;
major = sqrt(1 / A[2])
minor = sqrt(1 / A[0])
else:
R = ((A[2] - A[0]) / A[1])
tg = R - sqrt((R * R) + 1)
angle = math.atan(tg)
P = (2 * tg) / (1 + (tg * tg))
if ((A[0] > 0 and A[1] > 0 and A[2] > 0)):
if (angle < (-pi / 4)):
angle = angle + np.pi
else:
angle = angle
elif ((A[1] / P <= (-A[1] / P))):
if (angle < 0):
angle = angle + np.pi / 2
else:
angle = angle - np.pi / 2
elif (A[0] < 0 and A[1] < 0 and A[2] < 0):
if (angle < 0):
angle = angle + np.pi
else:
angle = angle - np.pi
else:
# Switch
if (angle < 0):
angle = angle + np.pi / 2
else:
angle = angle - np.pi / 2
t_New = (angle) #angle
t_new_test[n] = abs((angle* 180/np.pi)-90)-(2*angle* 180/np.pi)
Mirtheta1RadFinal1= Mirtheta1RadFinal_test[n]
#print('round2',Mirtheta1RadFinal1)
MirnormPower = MirnormPower_test[n]
#print('mirnormpower',MirnormPower)
t = t_new_test[n]
r_line = np.arange(0, max(MirnormPower) + .5, .5)
th = np.zeros(len(r_line))
for i in range(0, len(r_line)):
th[i] = t
th = np.concatenate([th, (th + 180)])
r_line = np.concatenate([r_line, r_line])
## Polar Graph
graphs = ["#71C4BE","#BBDBA0","#CE7E56","#BC5C81","#84355A"];
palette = seaborn.set_palette(graphs);
seaborn.set_context("poster",font_scale=1.5);
seaborn.set_style('ticks')
csfont = {'fontname':'Times New Roman',
'size':'14',
}
ticksfont = {'fontname':'Times New Roman'}
plt.figure(frameon=False, figsize=(15, 10));
plt.polar(Mirtheta1RadFinal1, abs(MirnormPower), 'o',ms=5);
plt.suptitle(abs(t)-180, fontsize=20);
plt.polar(th * pi/180, r_line,color="#CE7E56",linewidth=4);
plt.savefig(savepath+'polarpositive'+names[n]);
if (max(MirnormPower)<2):
inc = 0.5
elif (max(MirnormPower)<5):
inc = 1
elif max(MirnormPower)<20:
inc = 5
else:
inc = 10
plt.yticks(np.arange(inc, max(MirnormPower), inc));#, **ticksfont)
#plt.show()
t_final_rad = t*pi/180
def fitted_func(thetas, c):
int_value, int_err = scipy.integrate.quadrature(func=lambda x: exp(c * cos(x)), a=0.0, b=np.pi)
return ((np.pi * (1.0 / np.pi * (int_value))) ** - 1) * \
np.exp(c * np.cos(2 * (thetas - t_final_rad)))
c0 = 15
xdata = theta1RadFinal_test[n]
theta1RadFinal = theta1RadFinal_test[n]
ydata= normPower_test[n]
normPower = normPower_test[n]
kappa, kappa_pcov = scipy.optimize.curve_fit(f=fitted_func, p0=(c0,), xdata=theta1RadFinal_test[n], ydata=normPower_test[n])
# Shift data for plotting purposes
diff = abs(theta1RadFinal - (t * pi / 180))
#print(diff,'diff')
#centerLoc=diff.find(diff==min(diff))
centerLoc = diff[np.argmin(diff)]#find(diff== min(diff))
#print(centerLoc,'centerLoc')
num_angles = len(theta1RadFinal)
shift = ((round(num_angles / 2) - (num_angles - centerLoc)))
#print(shift,'shift')
normPower1 = np.roll(normPower, -int(shift))
normPower1
theta1RadFinal1 = np.roll(theta1RadFinal, -int(shift))
if (shift > 0):
s = num_angles - shift
s= np.asscalar(s)
for k in range(s, num_angles):
theta1RadFinal1[k] = np.pi + theta1RadFinal1[k]
elif (shift < 0):
for k in range(0,90):
theta1RadFinal1[k] = -np.pi + theta1RadFinal1[k]
## Normalized Distribution Graph
cartDist = plt.figure(figsize=(10, 10)); # Creates a figure containing cartesian distribution.
h2 = plt.bar((theta1RadFinal1 * 180 / pi), normPower1,color="#dedede");
plt.xticks(np.arange(-360, 360, 45,), **ticksfont);
plt.xlim([-95, 95]);
p_act = fitted_func(theta1RadFinal1, kappa);
#print(p_act)
a = np.vstack([[p_act],[(theta1RadFinal1 * 180 / pi)]]);
a= a.T
b = a[a[:,1].argsort()]#np.sort(a,axis=1)
h3, = plt.plot(b[:,1],b[:,0], linewidth=4,color="#CE7E56");
plt.xlabel('Angle (°)', **csfont);
plt.ylabel('Normalized Intensity', **csfont);
#print(max(normPower))
if (max(normPower)<2):
inc = 0.5
elif (max(normPower)<5):
inc = 1
elif (max(normPower)<20):
inc = 5
else:
inc = 10
plt.yticks(np.arange(0, max(normPower1) + .3, inc), **ticksfont);
plt.ylim([0, max(normPower1) + .3]);
normPower1_plot.append(normPower1);
plt.savefig(savepath+'Normdist '+names[n]);
#plt.show()
#cartDist.savefig(dir + 'cartDist_' + number.__str__(), bbox_inches='tight')
plt.close();
t_final=t
slope, intercept, rValue, pValue, stderr = scipy.stats.linregress(p_act, normPower1)
#print(rValue**2)
# Rounding results for Title of Figure
krnd = math.ceil(k * 1000) / 1000
thrnd = abs(math.ceil(t_final * 1000) / 1000)-180
krnd = math.trunc(krnd * 100) / 100
thrnd = math.trunc(thrnd * 100) / 100
a = 32.02
b= -12.43
c = 47.06
d = -0.9185
e = 19.43
f = -0.07693
x = kappa[0]
sig = math.exp(b*x) + c*math.exp(d*x) + e*exp(f*x)
#print("name",names[n],"kappa",kappa,"sigma",sig,"Rsq",rValue**2,"mu",t_final)
sum_NP = sum(normPower1)
#normPower2 = normPower1/sum(normPower1)
#sum_NP2 = sum(normPower2)
#print(sum_NP2)
## Combined plot figure
palette = seaborn.set_palette(graphs);
seaborn.set_context("notebook",font_scale=1.5);
seaborn.set_style('ticks')
csfont = {'fontname':'Times New Roman',
'size':'18',
}
ticksfont = {'fontname':'Times New Roman'}
fig = plt.figure(frameon=False,figsize=(20,12));
fig.subplots_adjust(top=0.95);
fig.suptitle('%s '%(names[n]), fontsize=34, fontweight='bold');
grid = plt.GridSpec(2, 5,hspace=0,wspace=0,width_ratios=[1, 1, 0.3, 1,1]);
cart_dist = plt.subplot(grid[1, 0:2]);
cart_dist.set_title('Von Mises Distribution (k)',fontsize=20);
h2 = cart_dist.bar((theta1RadFinal1 * 180 / pi), normPower1,color="#dedede");
cart_dist.set_xlim([-92, 92]);
cart_dist.set_xlabel('Angle (°)',fontsize=18);
cart_dist.set_ylim([0, max(normPower1) + .3]);
cart_dist.set_ylabel('Normalized Intensity',fontsize=18);
p_act = fitted_func(theta1RadFinal1, kappa)
a = np.vstack([[p_act],[(theta1RadFinal1 * 180 / pi)]])
a= a.T
b = a[a[:,1].argsort()]#np.sort(a,axis=1)
h3, = cart_dist.plot(b[:,1],b[:,0], linewidth=4,color="#CE7E56");
if (max(normPower)<2):
inc = 0.5
elif (max(normPower)<5):
inc = 1
elif (max(normPower)<20):
inc = 5
else:
inc = 10
polar_dist = plt.subplot(grid[0:2,3:5],projection='polar');
polar_dist.plot(Mirtheta1RadFinal1, abs(MirnormPower), 'o');
polar_dist.plot(th * pi/180, r_line,color="#CE7E56",linewidth=4);
polar_dist.set_title('Ellipse Fit (u)',fontsize=20).set_position([.5, 1.08]);
fig.text(0.75, 0.05, ' \n Kappa: %s '%(krnd)+' mu: %s \n'%(round((abs(t_final)-180),2))+' R^2: %s '%(round((rValue**2),2))+' sigma: %s \n'%(round(sig,2)),fontsize=20, fontweight='bold', bbox={'facecolor': '#BC5C81', 'alpha': 0.5, 'pad': 0.4});
#polar_dist.text(np.radians(-110),1.95, ' \n Kappa: %s '%(krnd)+' mu: %s \n'%(round((abs(t_final)-180),2))+' R^2: %s '%(round((rValue**2),2))+' sigma: %s \n'%(round(sig,2)),fontsize=20, fontweight='bold', bbox={'facecolor': '#BC5C81', 'alpha': 0.5, 'pad': 0.4})
orig_fig = plt.subplot(grid[0, 0]);
orig_fig.set_title('Original Image',fontsize=20).set_position([.5, 1.0]);
orig_fig.axis('off');
fft_fig = plt.subplot(grid[0, 1]);
fft_fig.set_title('Fast Fourier Transform',fontsize=20).set_position([.5, 1.0]);
fft_fig.set_axis_off();
fft_fig.axis('off');
#fft_fig.imshow(log(PabsFlip), cmap='gray')
#orig_fig.imshow(images[n],cmap=plt.cm.viridis )
#plt.tight_layout()
plt.savefig(savepath+'Grouped'+names[n]);
#print(n,'n')
#plt.show()
polar_dist_2 = plt.figure(figsize=(10, 10));
#print('linebeforeindex')
#print(index,'currentindex')
#Mirtheta1RadFinal1_plot[index]=Mirtheta1RadFinal1;
#MirnormPower_plot[index]=MirnormPower;
#th_plot[index] = th * pi/180
#print(th_plot[index],'thplotindex')
#r_line_plot[index]=r_line
Mirtheta1RadFinal1_plot.append(Mirtheta1RadFinal1);
MirnormPower_plot.append(MirnormPower);
th_plot.append(th * pi/180)
r_line_plot.append(r_line)
polar_dist.plot(Mirtheta1RadFinal1, MirnormPower);
polar_dist.plot(th * pi/180, r_line,linewidth=4);
polar_dist.set_title('Ellipse Fit (u)',fontsize=20).set_position([.5, 1.08]);
plt.close('all')
#index=index+1
#print(index,'Index')
## export of cartesian coordinate data
arr = numpy.array(MirnormPower_plot)
arr2 = numpy.array(Mirtheta1RadFinal1_plot)
arr3 = numpy.array(normPower1_plot)
arr4 = numpy.array(theta1RadFinal1*180/pi)
a = numpy.transpose(arr3)
b = numpy.transpose(arr4)
b[b> 100] = 0
b[b<-100] = 0
export_data=np.hstack(((b[:,None]),a))
numpy.savetxt(savepath+"data_export.csv", export_data, delimiter=",")
return export_data
def status_printer(file, total=None, desc=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook, tqdm_notebook).status_printer(file)
fp = file
if not getattr(fp, 'flush', False): # pragma: no cover
fp.flush = lambda: None
# Prepare IPython progress bar
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
pbar.description = desc
# Prepare status text
ptext = HTML()
# Only way to place text to the right of the bar is to use a container
container = HBox(children=[pbar, ptext])
display(container)
def print_status(s='', close=False, bar_style=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
container.visible = False
return print_status
def status_printer(file, total=None, desc=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook, tqdm_notebook).status_printer(file)
fp = file
if not getattr(fp, 'flush', False): # pragma: no cover
fp.flush = lambda: None
# Prepare IPython progress bar
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
pbar.description = desc
# Prepare status text
ptext = HTML()
# Only way to place text to the right of the bar is to use a container
container = HBox(children=[pbar, ptext])
display(container)
def print_status(s='', close=False, bar_style=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
container.visible = False
return print_status
def log_progress(sequence, every=None, size=None, with_time=False, name=''):
from ipywidgets import IntProgress, HTML, HBox
from IPython.display import display
from datetime import datetime
def format_time(time_value):
return str(time_value).split(".")[0]
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
with_time = False
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = HBox(children=[label, progress])
display(box)
time_started = datetime.now()
index = 0
try:
for index, record in enumerate(sequence, 0):
if index % every == 0:
if is_iterator:
label.value = '{name} {index} / ?'.format(name=name,
index=index)
else:
if with_time:
time_elapsed = datetime.now() - time_started
time_left = size / index * time_elapsed - time_elapsed
time_status = "| ETA: {}".format(
format_time(time_left))
else:
time_status = ""
progress.value = index
label.value = u'{name}: {index} / {size} {time}'.format(
name=name, index=index, size=size, time=time_status)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
index += 1
progress.value = index
label.value = "{name} {index} / {size} | Time elapsed {time_elapsed}".format(
name=name,
index=str(index or '?'),
size=size,
time_elapsed=format_time(time_started - datetime.now()),
)
