def log_progress(sequence, every=None, size=None):
is_iterator = False
start_tic = time.time()
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(index=index,
size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
# pretty human readable time diff
if False:
import dateutil.relativedelta
attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']
delta = dateutil.relativedelta.relativedelta(seconds=time.time() -
start_tic)
elapsed = " ".join([
'%d %s' % (getattr(delta, attr),
getattr(delta, attr) > 1 and attr or attr[:-1])
for attr in attrs if getattr(delta, attr)
]) + " %d usec" % ((tdiff - int(tdiff)) * 1000000)
label.value = u'{index} : {elapsed}'.format(index=index or '?',
elapsed=elapsed)
else:
# simple time in sec
label.value = u'{index} : {elapsed:0.2f}s'.format(
index=index or '?', elapsed=time.time() - start_tic)
def loading_bar_ipython(loading, data=None):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
progress = IntProgress(min=0, max=100, value=0)
progress.bar_style = 'info'
label = HTML()
box = VBox(children=[label, progress])
display(box)
page_cnt = 0
shown_categories = set()
while True:
if data and 'pages' in data:
page_cnt = len(data['pages'])
if data and 'categories' in data:
new_cats = set(data['categories'].keys())
for new in new_cats.difference(shown_categories):
print(f"\rCategory title: {new}")
shown_categories = new_cats
label.value = f'Pages: {page_cnt} / ?'
progress.value = page_cnt % 95
if not loading['status']:
label.value = f'Pages: {page_cnt} / {page_cnt}'
progress.value = 100
print(f"Scraping Done! {page_cnt} pages read")
break
time.sleep(0.5)
def log_progress(sequence, every=None, size=None, name='Items'):
"""Ipywidget for displaying a progress bar. Developed by Elisa Heim
Args:
sequence ([type]): [description]
every ([type], optional): [description]. Defaults to None.
size ([type], optional): [description]. Defaults to None.
name (str, optional): [description]. Defaults to 'Items'.
Yields:
Ipywidget: a progress bar showing current / max number of models.
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def log_progress(sequence, every=None, size=None, desc='Items'):
"""
Make and display a progress bar.
Parameters
----------
sequence : list.
Represents a elements sequence.
every : int, optional, default None.
Represents the steps in which the bar will be updated
size : int, optional, default None.
Represents the size/number elements in sequence.
desc : String, optional, default 'Items'.
Represents the description of the operation.
"""
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200)
else:
if every is None:
raise AssertionError('Sequence is iterator, set every')
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '%s: %s / ?' % (desc, index)
else:
progress.value = index
label.value = u'%s: %s / %s' % (desc, index, size)
yield record
except Exception:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = '%s: %s' % (desc, str(index or '?'))
def _log_progress(sequence, desc='Items', total=None, miniters=None):
"""
Make and display a progress bar.
Parameters
----------
sequence : list.
Represents a elements sequence.
desc : String, optional, default 'Items'.
Represents the description of the operation.
total : int, optional, default None.
Represents the total/number elements in sequence.
miniters : int, optional, default None.
Represents the steps in which the bar will be updated
"""
is_iterator = False
if total is None:
try:
total = len(sequence)
except TypeError:
is_iterator = True
if total is not None:
if miniters is None:
if total <= 200:
miniters = 1
else:
miniters = int(total / 200)
else:
if miniters is None:
miniters = 1
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=total, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % miniters == 0:
if is_iterator:
label.value = '%s: %s / ?' % (desc, index)
else:
progress.value = index
label.value = u'%s: %s / %s' % (desc, index, total)
yield record
except Exception:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = '%s: %s' % (desc, str(index or '?'))
def progressBar(sequence,
width=None,
every=None,
size=None,
name='Items',
text_color='white'):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=0, width=width)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0, width=width)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 0):
if index == 0 or index % every == 0:
if is_iterator:
label.value = '<font color="{color}">{name}: {index} / ?</font>'.format(
color=text_color, name=name, index=index)
else:
progress.value = index
label.value = u'<font color="{color}">{name}: {index} / {size}</font>'.format(
color=text_color, name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index + 1
label.value = u'<font color="{color}">{name}: {index} / {size}</font>'.format(
color=text_color,
name=name,
index=str(index + 1 or '?'),
size=size)
def log_progress(sequence, every=None, size=None, name='Items'):
"""https://github.com/alexanderkuk/log-progress
Arguments:
sequence {[type]} -- [description]
Keyword Arguments:
every {[type]} -- [description] (default: {None})
size {[type]} -- [description] (default: {None})
name {str} -- [description] (default: {'Items'})
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def log_progress(sequence, every=None, size=None, name='Items'):
"""
Displays a progress indicator
Example
def dothing(v):
time.sleep(1)
s = [1, 2, 3,4]
for r in log_progress(s, every=1):
dothing(r)
From https://github.com/alexanderkuk/log-progress
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def create_movie(self,
path,
size,
image_format='png',
animation_frames=list(),
quality=100,
samples_per_pixel=1,
start_frame=0,
end_frame=0,
interpupillary_distance=0.0,
exportIntermediateFrames=True):
from ipywidgets import IntProgress
application_params = self._client.get_application_parameters()
renderer_params = self._client.get_renderer()
old_image_stream_fps = application_params['image_stream_fps']
old_viewport_size = application_params['viewport']
old_samples_per_pixel = renderer_params['samples_per_pixel']
old_max_accum_frames = renderer_params['max_accum_frames']
self._client.set_renderer(samples_per_pixel=1,
max_accum_frames=samples_per_pixel)
self._client.set_application_parameters(viewport=size)
self._client.set_application_parameters(image_stream_fps=0)
progress_widget = IntProgress(description='In progress...',
min=0,
max=100,
value=0)
display(progress_widget)
self.export_frames(path=path,
animation_frames=animation_frames,
start_frame=start_frame,
end_frame=end_frame,
size=size,
samples_per_pixel=samples_per_pixel,
quality=quality,
interpupillary_distance=interpupillary_distance,
exportIntermediateFrames=exportIntermediateFrames)
done = False
while not done:
import time
time.sleep(1)
progress = self.get_export_frames_progress()['progress']
progress_widget.value = progress * 100
done = self.get_export_frames_progress()['done']
self._client.set_application_parameters(
image_stream_fps=old_image_stream_fps, viewport=old_viewport_size)
self._client.set_renderer(samples_per_pixel=old_samples_per_pixel,
max_accum_frames=old_max_accum_frames)
progress_widget.description = 'Done'
progress_widget.value = 100
def progress(sequence, every=None, size=None, name='Items'):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
import time
import datetime
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
start = time.time()
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
elapsed = time.time() - start
avg_iter = elapsed / index
remaining = int(avg_iter * (size - index))
label.value = u'{name}: {index} / {size} [ETA: {seconds}]'.format(
name=name,
index=index,
size=size,
seconds=datetime.timedelta(seconds=remaining))
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
def log_progress(sequence, every=None, size=None):
is_iterator = False
start_tic = time.time()
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
# pretty human readable time diff
if False:
import dateutil.relativedelta
attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']
delta = dateutil.relativedelta.relativedelta(seconds=time.time()-start_tic)
elapsed = " ".join(['%d %s' % (getattr(delta, attr),
getattr(delta, attr) > 1 and attr or attr[:-1]) for attr in attrs if
getattr(delta, attr)])+" %d usec"%((tdiff-int(tdiff))*1000000)
label.value = u'{index} : {elapsed}'.format(index=index or '?', elapsed=elapsed)
else:
# simple time in sec
label.value = u'{index} : {elapsed:0.2f}s'.format(index=index or '?', elapsed=time.time()-start_tic)
def _log_progress(sequence, every=1, size=None, name='Position', timeit=True):
from ipywidgets import IntProgress, HTML, VBox, HBox
from IPython.display import display
progress_logger = logging.getLogger(__name__ + ".progress")
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
timeit = False
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
position_label = HTML()
time_label = HTML()
label = HBox(children=[position_label, time_label])
box = VBox(children=[label, progress])
display(box)
index = 0
if timeit:
t_remaining = "?"
t_start = timer()
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
position_label.value = u'{}: {} / ?'.format(name, index)
else:
progress.value = index
position_label.value = u'{}: {} / {}'.format(name, index, size)
if timeit:
time_label.value = u' | Remaining: {}'.format(_format_remaining(t_remaining))
progress_logger.info(position_label.value + time_label.value)
yield record
if timeit:
t_remaining = (size - index - 1) * (timer() - t_start) / (index)
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
position_label.value = "{}: {}".format(name, str(index or '?'))
def create_snapshot(self, size, path, samples_per_pixel, export_intermediate_frames=False):
"""
Create a snapshot of the current frame
:size: Frame buffer size
:path: Full path of the snapshot image
:samples_per_pixel: Samples per pixel
:export_intermediate_frames: If True, intermediate samples are stored to disk. Otherwise, only
the final accumulation is exported
"""
application_params = self._client.get_application_parameters()
renderer_params = self._client.get_renderer()
old_image_stream_fps = application_params['image_stream_fps']
old_viewport_size = application_params['viewport']
old_samples_per_pixel = renderer_params['samples_per_pixel']
old_max_accum_frames = renderer_params['max_accum_frames']
old_smoothed_key_frames = copy.deepcopy(self._smoothed_key_frames)
self._client.set_renderer(samples_per_pixel=1, max_accum_frames=samples_per_pixel)
self._client.set_application_parameters(viewport=size)
self._client.set_application_parameters(image_stream_fps=0)
control_points = [self.get_camera()]
current_animation_frame = int(self._client.get_animation_parameters()['current'])
animation_frames = [current_animation_frame]
self.build_camera_path(
control_points=control_points, nb_steps_between_control_points=1, smoothing_size=1)
progress_widget = IntProgress(description='In progress...', min=0, max=100, value=0)
display(progress_widget)
base_dir = os.path.dirname(path)
self.export_frames(
path=base_dir, animation_frames=animation_frames, size=size,
samples_per_pixel=samples_per_pixel,
export_intermediate_frames=export_intermediate_frames)
done = False
while not done:
time.sleep(1)
progress = self.get_export_frames_progress()['progress']
progress_widget.value = progress * 100
done = self.get_export_frames_progress()['done']
progress_widget.description = 'Done'
progress_widget.value = 100
frame_path = base_dir + '/00000.png'
if os.path.exists(frame_path):
os.rename(frame_path, path)
self._client.set_application_parameters(image_stream_fps=old_image_stream_fps,
viewport=old_viewport_size)
self._client.set_renderer(samples_per_pixel=old_samples_per_pixel,
max_accum_frames=old_max_accum_frames)
self._smoothed_key_frames = copy.deepcopy(old_smoothed_key_frames)
def progress_bar(sequence, every=None, size=None, name='Progress'):
'''Source: https://github.com/kuk/log-progress. Makes progress bar during iterating through sequence. Example:
from time import sleep
for i in progress_bar([1,2,3,4],every=1):
sleep(1)'''
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
def log_progress(sequence, every=None, size=None, name='Items') -> object:
"""
Shows a progress bar with labels in a Jupyter notebook (see https://github.com/kuk/log-progress).
Args:
sequence: The list to iterate over. Each element in the list can cause a progress bar update but the frequency depends on the every parameter.
every: The frequency of the progress bar update. E.g. update progress bar after every two items.
size: The number of items in the list.
name: The description to show.
Returns:
"""
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = f'{name}: {record} / ?'
else:
progress.value = index
label.value = f'{name}: {record} / {sequence[-1]}'
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = f'{name}: {record} / {sequence[-1]}'
def log_progress(sequence, every=None, size=None, name='Items'):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
yield record
except Exception:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
def log_progress(sequence, every=None, size=None):
"""
Returns a log progress bar for computations
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = str(index or '?')
def log_progress(sequence, every=None, size=None):
"""Widget based progress bar for Jupyter (IPython Notebook)
Author: Kukushkin Alexander
Source: https://github.com/alexanderkuk/log-progress
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = str(index or '?')
def log_progress(sequence, every=None, size=None, name='Items'):
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
def create_movie(
self, path, size, animation_frames=list(), quality=100, samples_per_pixel=1,
start_frame=0, end_frame=0, interpupillary_distance=0.0, exportIntermediateFrames=True):
"""
Create and export a set of PNG frames for later movie generation
:path: Full path of the snapshot folder
:size: Frame buffer size
:animation_frames: Optional list of animation frames
:quality: PNG quality
:samples_per_pixel: Samples per pixel
:start_frame: Start frame to export in the provided sequence
:end_frame: Last frame to export in the provided sequence
:interpupillary_distance: Interpupillary distance for stereo rendering. If set to 0, stereo
is disabled
:exportIntermediateFrames: If True, intermediate samples are stored to disk. Otherwise, only
the final accumulation is exported
"""
application_params = self._client.get_application_parameters()
renderer_params = self._client.get_renderer()
old_image_stream_fps = application_params['image_stream_fps']
old_viewport_size = application_params['viewport']
old_samples_per_pixel = renderer_params['samples_per_pixel']
old_max_accum_frames = renderer_params['max_accum_frames']
self._client.set_renderer(samples_per_pixel=1, max_accum_frames=samples_per_pixel)
self._client.set_application_parameters(viewport=size)
self._client.set_application_parameters(image_stream_fps=0)
progress_widget = IntProgress(description='In progress...', min=0, max=100, value=0)
display(progress_widget)
self.export_frames(
path=path, animation_frames=animation_frames, start_frame=start_frame,
end_frame=end_frame, size=size, samples_per_pixel=samples_per_pixel, quality=quality,
interpupillary_distance=interpupillary_distance,
exportIntermediateFrames=exportIntermediateFrames)
done = False
while not done:
time.sleep(1)
progress = self.get_export_frames_progress()['progress']
progress_widget.value = progress * 100
done = self.get_export_frames_progress()['done']
self._client.set_application_parameters(image_stream_fps=old_image_stream_fps,
viewport=old_viewport_size)
self._client.set_renderer(samples_per_pixel=old_samples_per_pixel,
max_accum_frames=old_max_accum_frames)
progress_widget.description = 'Done'
progress_widget.value = 100
def log_progress(sequence, every=None, size=None, name='Items'):
"""
Widget based progress bar for Jupyter, source: https://github.com/kuk/log-progress
(requires the jupyter lab extension to be installed: $jupyter labextension install @jupyter-widgets/jupyterlab-manager)
"""
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def progress_bar(sequence, every=None, size=None, name='Progress'):
'''Source: https://github.com/kuk/log-progress. Makes progress bar during iterating through sequence. Example:
from time import sleep
for i in progress_bar([1,2,3,4],every=1):
sleep(1)'''
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def log_progress(sequence, every=None, size=None, name="Items"):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, "sequence is iterator, set every"
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = "info"
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = "{name}: {index} / ?".format(name=name,
index=index)
else:
progress.value = index
label.value = u"{name}: {index} / {size}".format(
name=name, index=index, size=size)
yield record
except Exception:
progress.bar_style = "danger"
raise
else:
progress.bar_style = "success"
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or "?"))
def combine(df, df_):
progress = IntProgress(min=0, max=len(df_), value=0)
display(progress)
for i in range(len(df_)):
player1 = df_['player1'][i]
player2 = df_['player2'][i]
date = np.datetime64(df_['date'][i])
match = df[(df['player1'] == player1) & (df['player2'] == player2) &
((df['date'] == date) | (df['date'] == date + 1) |
(df['date'] == date - 1))]
if len(match) == 1:
for k in list(df.keys())[3:]: ## k1_1Xbet, ....
df_.loc[i, k] = float(match[k])
match = df[(df['player1'] == player2) & (df['player2'] == player1) &
((df['date'] == date) | (df['date'] == date + 1) |
(df['date'] == date - 1))]
if len(match) == 1:
for k in list(df.keys())[3:]: ## k1_1Xbet, ....
if 'k1' in k:
df_.loc[i, k.replace('k1', 'k2')] = float(match[k])
else:
df_.loc[i, k.replace('k2', 'k1')] = float(match[k])
if (i % 10 == 0):
progress.value = i
def perceptron_pass2(lines, classifier):
n = len(lines)
count = 0
p = IntProgress(max=n)
display(p)
c = classifier
random.shuffle(lines)
for line in lines:
count += 1
label, feature = get_label_feature(line)
value = get_value(classifier, feature)
if label * value > 0:
pass
else:
for word in feature:
if None == classifier.get(word):
classifier[word] = label * feature[word]
c[word] = label * feature[word] * (n - count) / n
else:
classifier[word] += label * feature[word]
c[word] += label * feature[word] * (n - count) / n
p.value = count
p.description = "{}%".format(round(100 * p.value / p.max, 2))
p.description = "Done"
return c
def tile_image(full_size_path, tile_size=400):
p = IntProgress(description='Tiling', max=1)
display(p)
root_dir = os.path.dirname(full_size_path)
name, ext = os.path.splitext(os.path.basename(full_size_path))
tile_dir = os.path.join(root_dir, '{0}-{1}x{1}'.format(name, tile_size))
if not os.path.exists(tile_dir):
os.mkdir(tile_dir)
tpl = os.path.join(tile_dir, '{name}-{i:02}-{j:02}{ext}')
full_size = skimage.io.imread(full_size_path)
X, Y = full_size.shape[:2]
total_tiles = (X // tile_size) * (Y // tile_size)
p.max = total_tiles
print("Creating %i tiles in %s" % (total_tiles, tile_dir))
for i in range(X // tile_size):
for j in range(Y // tile_size):
tile = full_size[
i * tile_size: (i+1) * tile_size,
j * tile_size: (j+1) * tile_size,
:
]
fname = tpl.format(**locals())
p.value += 1
skimage.io.imsave(fname, tile)
p.value = p.max
def progress_bar(job):
'''Display a status bar showing how many tasks are completed'''
status=get_status(job)
f = IntProgress(min=0, max=status['Total'], bar_style='success')
f.value = status['Total'] - status['Pending'] - status['Running']
f.description = "{:1.0f} tasks left ".format(status['Pending']+status['Running'])
display(f)
while f.value < status['Total']:
status=get_status(job)
f.value = status['Total'] - status['Pending'] - status['Running']
f.description = "{:1.0f} tasks left ".format(status['Pending']+status['Running'])
if status['Failed'] > 0:
f.bar_style='warning'
sleep(1)
def log_progress(sequence, every=None, size=None):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, msg in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = 'Running: {index} / ?: {msg}...'.format(
index=index, msg=msg)
else:
progress.value = index
label.value = u'Running: {index} / {size}: {msg}...'.format(
index=index, size=size, msg=msg)
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = size
# label.value = str(index or '?')
label.value = 'Done.'
def sample(self,
T=20000,
store_every=10,
burn_in=1000,
report_every=100,
tune_every=50):
print('----- Metropolis Begins -----')
self.acceptance_rates = {param.name: 0.
for param in self.params
} # Reset acceptance rates
f = IntProgress(description='Running', min=0,
max=T) # instantiate the bar
display(f)
for iteration_number in range(T):
if iteration_number % report_every == 0:
f.value = iteration_number
if iteration_number >= 1 and iteration_number % tune_every == 0:
for param in self.params:
acc = self.acceptance_rates[param.name] / iteration_number
param.step_size = self.tune(param.step_size, acc)
#print(f'Updating {param.name} to {param.step_size} due to acceptance rate {acc}')
self.iterate()
if iteration_number >= burn_in and iteration_number % store_every == 0:
# for j in range(num_genes):
for param in self.params:
if param.value.ndim > 1:
if is_tensor(param.value):
self.samples[param.name].add(
param.value.numpy().copy())
else:
self.samples[param.name].add(param.value.copy())
else:
self.samples[param.name].add(param.value)
acc = self.acceptance_rates[param.name] / (
iteration_number if iteration_number > 0 else 1)
self.samples['acc_rates'][param.name].add(acc)
# for key in self.acceptance_rates:
# self.acceptance_rates[key] /= T
# rates = np.array(self.samples['acc_rates']).T/np.arange(1, T-burn_in+1, store_every)
# self.samples['acc_rates'] = rates
f.value = T
print('----- Finished -----')
def progress_bar(sequence, every=None, size=None, name='Parameters'):
from ipywidgets import IntProgress, HTML, VBox
# c&p from https://github.com/kuk/log-progress
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
IPython.display.display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = f"{name}: {str(index or '?')}"
def __get_max_gap_length(self, b):
"""
Compute the max gep length of a masked time series
:param b: Progress bar object
"""
# TODO
# This function should be paralelised!
bands, rows, cols = self.mask.shape
max_gap_length = np.zeros((rows, cols), np.int16)
if not type(b) == QProgressBar:
progress_bar = IntProgress(
value=0,
min=0,
max=10,
step=1,
description='Computing max gap length...',
bar_style='', # 'success', 'info', 'warning', 'danger' or ''
orientation='horizontal',
style = {'description_width': 'initial'},
layout={'width': '50%'}
)
display(progress_bar)
else:
b.setEnabled(True)
for i in range(rows):
if type(b) == QProgressBar:
b.setFormat('Computing maximum gap length...')
b.setValue(int((i*10.)/rows))
else:
progress_bar.value = int((i*10.)/rows)
for j in range(cols):
for key, group in i_groupby(self.mask.data[:,i,j]):
if key == False:
_gap_lenght = len(list(group))
if _gap_lenght > 0 and _gap_lenght > max_gap_length[i,j]:
max_gap_length[i,j] = _gap_lenght
if type(b) == QProgressBar:
b.setValue(0)
b.setEnabled(False)
else:
# Remove progress bar
progress_bar.close()
del progress_bar
# Create xarray DataArray
_max_gap_length = xr.DataArray(max_gap_length,
coords=[self.mask.latitude.data,
self.mask.longitude.data],
dims=['latitude', 'longitude'])
max_gap_length = None
self.max_gap_length = _max_gap_length
def add_stats(df, players, is_fst_time=1):
names = ['p1_height', 'p1_birth', 'p1_hand', 'p1_nation']
if is_fst_time:
print("IMPORTING STATIC STATS...")
progress = IntProgress(min=0, max=len(df), value=0)
display(progress)
if is_fst_time:
for name in names:
df[name] = [np.nan for x in range(len(df))]
for name in names:
df[name.replace('1', '2')] = [np.nan for x in range(len(df))]
arr1_player1 = []
arr1_player2 = []
arr2_player1 = []
arr2_player2 = []
arr3_player1 = []
arr3_player2 = []
arr4_player1 = []
arr4_player2 = []
for i in range(len(df)):
try:
player1 = players[df['player1'][i]:].head(1)
arr1_player1.append(player1['height'][0])
arr2_player1.append(player1['birth_date'][0])
arr4_player1.append(player1['hand'][0])
arr3_player1.append(player1['nationality'][0])
except Exception:
arr1_player1.append(df['p1_height'][i])
arr2_player1.append(df['p1_birth'][i])
arr4_player1.append(df['p1_hand'][i])
arr3_player1.append(df['p1_nation'][i])
try:
player2 = players[df['player2'][i]:].head(1)
arr1_player2.append(player2['height'][0])
arr2_player2.append(player2['birth_date'][0])
arr4_player2.append(player2['hand'][0])
arr3_player2.append(player2['nationality'][0])
except Exception:
arr1_player2.append(df['p2_height'][i])
arr2_player2.append(df['p2_birth'][i])
arr4_player2.append(df['p2_hand'][i])
arr3_player2.append(df['p2_nation'][i])
## Progress fill
if (i % 10 == 0):
progress.value = i
df['p1_height'] = arr1_player1
df['p2_height'] = arr1_player2
df['p1_birth'] = arr2_player1
df['p2_birth'] = arr2_player2
df['p1_nation'] = arr3_player1
df['p2_nation'] = arr3_player2
df['p1_hand'] = arr4_player1
df['p2_hand'] = arr4_player2
if not is_fst_time:
print("STATIC STATS IMPORTED\n")
def calcular_periodo_vulnerabilidad(inicio, fin, min_defunciones=-1):
inicio = pd.to_datetime(inicio, yearfirst=True)
fin = pd.to_datetime(fin, yearfirst=True)
fechas = pd.date_range(inicio, fin)
resultados = []
modelos = []
asegura_archivos_covid_disponibles(fechas)
f = IntProgress(min=0, max=len(fechas) - 1) # instantiate the bar
display(f) # display the bar
for count, fecha in enumerate(fechas):
covid_municipal = tabla_covid_indicadores_municipales(
fecha.strftime("%y%m%d"))
covid_municipal = agregar_tasas_municipales(covid_municipal)
caracteristicas = caracteristicas_modelos_municipios(covid_municipal)
pls = ajustar_pls_letalidad(covid_municipal,
caracteristicas,
min_defunciones=min_defunciones)
df = calificar_municipios_letalidad_formato_largo(covid_municipal,
pls,
caracteristicas,
modelo='PLS',
dia_ajuste=fecha)
resultados.append(df)
modelo = pd.DataFrame({
'caracteristica': caracteristicas,
'coef': pls.coef_
})
modelo['dia_ajuste'] = fecha
modelo['modelo'] = 'PLS'
modelos.append(modelo)
rf = ajustar_rf_letalidad(covid_municipal,
caracteristicas,
min_defunciones=min_defunciones)
df = calificar_municipios_letalidad_formato_largo(covid_municipal,
rf,
caracteristicas,
modelo='RF',
dia_ajuste=fecha)
resultados.append(df)
modelo = pd.DataFrame({
'caracteristica': caracteristicas,
'coef': rf.feature_importances_
})
modelo['dia_ajuste'] = fecha
modelo['modelo'] = 'RF'
modelos.append(modelo)
f.value = count
resultados_df = pd.concat(resultados, ignore_index=True)
modelos_df = pd.concat(modelos, ignore_index=True)
return modelos_df, resultados_df
def log_progress(sequence, name, every=None, size=None, who="reads"):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, "sequence is iterator, set every"
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = "info"
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = name + ": " + "{index} {who} / ?".format(index=index, who=who)
else:
progress.value = index
label.value = u"{name}: {index} / {size}".format(name=name, index=index, size=size)
yield record
except:
progress.bar_style = "danger"
raise
else:
progress.bar_style = "success"
progress.value = index
label.value = str(name + ": " + str(index) + " " + str(who) or "?")
def download_original():
p = IntProgress(max=1, description="Downloading")
display(p)
if os.path.exists(full_size_file):
print("Already have %s" % full_size_file)
p.value = p.max
else:
r = requests.get(url, stream=True)
content_length = r.headers.get('content-length', int(1e8))
print("Downloading %s" % url)
p.max = content_length
r.raise_for_status()
with open(full_size_file, 'wb') as f:
for chunk in r.iter_content(chunk_size=8096):
p.value += len(chunk)
f.write(chunk)
p.value = p.max
def screenshot(url: str, api_key: str = None) -> requests.models.Response:
"""
Get a screenshot of a url with Browshot.
Parameters
----------
url : str
The url a screenshot is wanted for.
api_key : str (optional)
Browshot API key. If not set msticpyconfig checked for this.
Returns
-------
image_data: requests.models.Response
The final screenshot request response data.
"""
# Get Broshot API key from kwargs or config
if api_key is not None:
bs_api_key = api_key
elif config.settings.get("Browshot") is not None:
bs_api_key = config.settings.get("Browshot")["Args"][
"AuthKey"] # type: ignore
else:
raise AttributeError("No configuration found for Browshot")
# Request screenshot from Browshot and get request ID
id_string = f"https://api.browshot.com/api/v1/screenshot/create?url={url}/&instance_id=26&size=screen&cache=0&key={bs_api_key}" # pylint: disable=line-too-long
id_data = requests.get(id_string)
bs_id = json.loads(id_data.content)["id"]
status_string = (
f"https://api.browshot.com/api/v1/screenshot/info?id={bs_id}&key={bs_api_key}"
)
image_string = f"https://api.browshot.com/api/v1/screenshot/thumbnail?id={bs_id}&zoom=50&key={bs_api_key}" # pylint: disable=line-too-long
# Wait until the screenshot is ready and keep user updated with progress
print("Getting screenshot")
progress = IntProgress(min=0, max=40)
display.display(progress)
ready = False
while ready is False:
progress.value += 1
status_data = requests.get(status_string)
status = json.loads(status_data.content)["status"]
if status == "finished":
ready = True
else:
time.sleep(0.05)
progress.value = 40
# Once ready get the screenshot
image_data = requests.get(image_string)
if image_data.status_code != 200:
print(
"There was a problem with the request, please check the status code for details"
)
return image_data
def in_progress(seq, msg="Progress: [%(processed)d / %(total)d]", length=None):
""" Iterate over sequence, yielding item with progress widget displayed.
This is useful if you need to precess sequence of items with some
time consuming operations
.. note::
This works only in Jupyter Notebook
.. note::
This function requires *ipywidgets* package to be installed
:param seq: sequence to iterate on.
:param str msg: (optional) message template to display.
available to use 'processed' and 'total' integer vars,
where 'processed' is number of items processed and
'total' is total number of items in seq.
:param int length: (optional) if seq is generator, or it is not
possible to apply 'len(seq)' function to 'seq',
then this argument is required and it's value will
be used as total number of items in seq.
Example example::
import time
for i in in_progress(range(10)):
time.sleep(1)
"""
from IPython.display import display
from ipywidgets import IntProgress
if length is None:
length = len(seq)
progress = IntProgress(value=0, min=0, max=length,
description=msg % {'processed': 0,
'total': length})
display(progress)
for i, item in enumerate(seq, 1):
progress.value = i
progress.description = msg % {'processed': i, 'total': length}
yield item
progress.close()
def dotrain(model, crit, aug, Xtr, ytr, nepochs=50, batchsize=100, title=None):
opt = df.AdaDelta(rho=0.95, eps=1e-7, lr=1)
progress = IntProgress(value=0, min=0, max=nepochs, description="Training:")
display(progress)
model.training()
costs = []
for e in range(nepochs):
batchcosts = []
for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True):
if aug is not None:
Xb, yb = aug.augbatch_train(Xb, yb)
model.zero_grad_parameters()
cost = model.accumulate_gradients(Xb, yb, crit)
opt.update_parameters(model)
batchcosts.append(cost)
costs.append(np.mean(batchcosts))
progress.value = e + 1
liveplot(plotcost, costs, title)
return costs
def int_progress(min_, max_):
prog = IntProgress(min=min_, max=max_)
display(prog)
for i in linspace(min_, max_, 25):
time.sleep(0.1)
prog.value = i
def display_multi_progress(results, objectlist, sleep=1):
prog = IntProgress(min=0, max=len(list(objectlist))-1)
display(prog)
while not results.ready():
prog.value = results.progress
time.sleep(sleep)
def animator(progbar,
count,
tot_string=False,
linenum=False,
terminal=False,
init=False,
length=False,
**kwargs
):
"""
Animates progress bar in unique position in terminal
Multiple progress bars not supported in jupyter yet.
"""
# if ipython?
welcome_message = kwargs.pop('welcome_message', '')
if welcome_message:
welcome_message = welcome_message.replace('Interrogating corpus ... \n', '')
welcome_message = welcome_message.replace('Concordancing corpus ... \n', '')
welcome_message = welcome_message.replace('\n', '<br>').replace(' ' * 17, ' ' * 17)
else:
welcome_message = ''
if init:
from traitlets import TraitError
try:
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
progress = IntProgress(min=0, max=length, value=1)
using_notebook = True
progress.bar_style = 'info'
label = HTML()
label.font_family = 'monospace'
gblabel = HTML()
gblabel.font_family = 'monospace'
box = VBox(children=[label, progress, gblabel])
display(box)
return box
except TraitError:
pass
except ImportError:
pass
if not init:
try:
from ipywidgets.widgets.widget_box import FlexBox
if isinstance(progbar, FlexBox):
label, progress, goodbye = progbar.children
progress.value = count
if count == length:
progress.bar_style = 'success'
else:
label.value = '%s\nInterrogating: %s ...' % (welcome_message, tot_string)
return
except:
pass
# add startnum
start_at = kwargs.get('startnum', 0)
if start_at is None:
start_at = 0.0
denominator = kwargs.get('denom', 1)
if kwargs.get('note'):
if count is None:
perc_done = 0.0
else:
perc_done = (count * 100.0 / float(length)) / float(denominator)
kwargs['note'].progvar.set(start_at + perc_done)
kwargs['root'].update()
return
if init:
from corpkit.textprogressbar import TextProgressBar
return TextProgressBar(length, dirname=tot_string)
# this try is for sublime text nosetests, which don't take terminal object
try:
with terminal.location(0, terminal.height - (linenum + 1)):
if tot_string:
progbar.animate(count, tot_string)
else:
progbar.animate(count)
# typeerror for nose
except:
if tot_string:
progbar.animate(count, tot_string)
else:
progbar.animate(count)
def eoldas_inversion ( fluxnet_site, year, green_leaves, gamma_lai,
albedo_unc_avg, albedo_unc_good,
n_tries=10 ):
f = IntProgress(min=0, max=2*n_tries + 1)
f.value = 1
display(f)
# convert uncertainties from percent
albedo_unc = [ albedo_unc_good*0.01, albedo_unc_avg*0.01 ]
retval_s, state, obs = tip_inversion( year, fluxnet_site, albedo_unc=albedo_unc,
green_leaves=green_leaves,
n_tries=n_tries, progressbar=f )
mu = state.operators['Prior'].mu
cinv = state.operators['Prior'].inv_cov
c = np.array(np.sqrt(np.linalg.inv (cinv.todense()).diagonal())).squeeze()
post_sd = np.sqrt(np.array(retval_s['post_cov'].todense()).squeeze())
post_sd_single = np.where(post_sd > c, c, post_sd)
retval, state, obs = regularised_tip_inversion( year, fluxnet_site, [1e-3, 0, 0.1, 1e-3, 0, 0.1, gamma_lai ],
albedo_unc=albedo_unc,
x0=retval_s['real_map'], green_leaves=green_leaves,
n_tries=n_tries, progressbar=f)
mu = state.operators['Prior'].mu
cinv = state.operators['Prior'].inv_cov
c = np.array(np.sqrt(np.linalg.inv (cinv.todense()).diagonal())).squeeze()
post_sd = np.sqrt(np.array(retval_s['post_cov'].todense()).squeeze())
post_sd = np.where(post_sd > c, c, post_sd)
fig, axs = plt.subplots(nrows=5, ncols=2, figsize=(14, 12))
axs = axs.flatten()
fig.suptitle("%s (%d)" % (fluxnet_site, year), fontsize=18)
params = ['omega_vis', 'd_vis', 'a_vis', 'omega_nir', 'd_nir', 'a_nir', 'lai']
post_sd = np.sqrt(np.array(retval['post_cov'].todense()).squeeze())
post_sd = np.where(post_sd > c, c, post_sd)
for i, p in enumerate(tip_params):
#axs[i].axhspan(mu[(i*46):((i+1)*46)][0]+c[(i*46):((i+1)*46)][0],
# mu[(i*46):((i+1)*46)][0] - c[(i * 46):((i + 1) * 46)][0], color="0.9" )
axs[i].fill_between ( state.state_grid, retval['real_map'][params[i]] - post_sd[(i*46):((i+1)*46)],
retval['real_map'][params[i]] + post_sd[(i*46):((i+1)*46)], lw=0.8, color="0.8")
axs[i].vlines ( state.state_grid, retval_s['real_map'][params[i]] - post_sd_single[(i*46):((i+1)*46)],
retval_s['real_map'][params[i]] + post_sd_single[(i*46):((i+1)*46)], lw=0.8,
colors="0.1", alpha=0.5)
axs[i].plot(state.state_grid, retval['real_map'][params[i]], 'o-', mfc="none")
axs[i].plot(state.state_grid, retval_s['real_map'][params[i]], '--')
if i in [ 1, 4, 6]:
axs[i].set_ylim(0, 6)
else:
axs[i].set_ylim(0, 1)
axs[i].set_ylabel( tip_params[i] )
fwd = np.array(obs.fwd_modelled_obs)
axs[7].plot(obs.observations[:, 0], fwd[:, 0], 'k+', label="VIS")
axs[7].plot(obs.observations[:, 1], fwd[:, 1], 'rx', label="NIR")
axs[7].set_xlabel("Measured BHR [-]")
axs[7].set_ylabel("Predicted BHR [-]")
axs[7].plot ( [0,0.9], [0, 0.9], 'k--', lw=0.5)
axs[7].legend(loc='best')
axs[8].vlines(obs.mask[:, 0], obs.observations[:, 0] - 1.96 * obs.bu[:, 0],
obs.observations[:, 0] + 1.96 * obs.bu[:, 0])
axs[8].plot(obs.mask[:, 0], obs.observations[:, 0], 'o')
axs[9].vlines(obs.mask[:, 0], obs.observations[:, 1] - 1.96 * obs.bu[:, 1],
obs.observations[:, 1] + 1.96 * obs.bu[:, 1])
axs[9].plot(obs.mask[:, 0], obs.observations[:, 1], 'o')
axs[8].set_ylabel("BHR VIS [-]")
axs[9].set_ylabel("BHR NIR [-]")
axs[8].set_xlabel("DoY [d]")
axs[9].set_xlabel("DoY [d]")
for i in xrange(10):
if i != 7:
axs[i].set_xlim(1, 370)
# Hide the right and top spines
axs[i].spines['right'].set_visible(False)
axs[i].spines['top'].set_visible(False)
# Only show ticks on the left and bottom spines
axs[i].yaxis.set_ticks_position('left')
axs[i].xaxis.set_ticks_position('bottom')
fig.figimage(logo, fig.bbox.xmax - 500, fig.bbox.ymax - 250, alpha=.4, zorder=1)
plt.savefig("regularised_model_%s_%04d_%02dpcnt_%02dpcnt.pdf" % (fluxnet_site, year,
int(albedo_unc[0]*100.), int(albedo_unc[1]*100.)),
dpi=300, bbox_inches="tight")
def eoldas_inversion ( fluxnet_site, year, n_years, green_leaves, gamma_lai,
n_tries=5 ):
f = IntProgress(min=0, max=2*n_tries + 1)
f.value = 1
display(f)
prior_set = []
for train_year in xrange ( n_years ):
this_year = (year - n_years - 1) + train_year
retval_s, state, obs = tip_inversion( this_year, fluxnet_site, green_leaves=green_leaves,
n_tries=2 )
prior_set.append ( retval_s )
mu_priors = np.array( [ state.__unpack_from_dict (p['real_map']) for p in prior_set ] )
cov_mtx = np.zeros (( 322, 322))
for p in prior_set:
cov_mtx = cov_mtx + p['hessian'].todense()
prior = Prior ( mu_priors.mean(axis=0), cov_mtx )
mu = state.operators['Prior'].mu
cinv = state.operators['Prior'].inv_cov
c = np.array(np.sqrt(np.linalg.inv (cinv.todense()).diagonal())).squeeze()
post_sd = np.sqrt(np.array(retval_s['post_cov'].todense()).squeeze())
post_sd_single = np.where(post_sd > c, c, post_sd)
retval, state, obs = regularised_tip_inversion( year, fluxnet_site, [1e-3, 0, 0.1, 1e-3, 0, 0.1, gamma_lai ],
x0=retval_s['real_map'], green_leaves=green_leaves,
n_tries=n_tries, progressbar=f)
mu = state.operators['Prior'].mu
cinv = state.operators['Prior'].inv_cov
c = np.array(np.sqrt(np.linalg.inv (cinv.todense()).diagonal())).squeeze()
post_sd = np.sqrt(np.array(retval_s['post_cov'].todense()).squeeze())
post_sd = np.where(post_sd > c, c, post_sd)
fig, axs = plt.subplots(nrows=5, ncols=2, figsize=(14, 12))
axs = axs.flatten()
fig.suptitle("%s (%d)" % (fluxnet_site, year), fontsize=18)
params = ['omega_vis', 'd_vis', 'a_vis', 'omega_nir', 'd_nir', 'a_nir', 'lai']
post_sd = np.sqrt(np.array(retval['post_cov'].todense()).squeeze())
post_sd = np.where(post_sd > c, c, post_sd)
for i, p in enumerate(tip_params):
#axs[i].axhspan(mu[(i*46):((i+1)*46)][0]+c[(i*46):((i+1)*46)][0],
# mu[(i*46):((i+1)*46)][0] - c[(i * 46):((i + 1) * 46)][0], color="0.9" )
axs[i].fill_between ( state.state_grid, retval['real_map'][params[i]] - post_sd[(i*46):((i+1)*46)],
retval['real_map'][params[i]] + post_sd[(i*46):((i+1)*46)], lw=0.8, color="0.8")
axs[i].vlines ( state.state_grid, retval_s['real_map'][params[i]] - post_sd_single[(i*46):((i+1)*46)],
retval_s['real_map'][params[i]] + post_sd_single[(i*46):((i+1)*46)], lw=0.8,
colors="0.1", alpha=0.5)
axs[i].plot(state.state_grid, retval['real_map'][params[i]], 'o-', mfc="none")
axs[i].plot(state.state_grid, retval_s['real_map'][params[i]], '--')
if i in [ 1, 4, 6]:
axs[i].set_ylim(0, 6)
else:
axs[i].set_ylim(0, 1)
axs[i].set_ylabel( tip_params[i] )
fwd = np.array(obs.fwd_modelled_obs)
axs[7].plot(obs.observations[:, 0], fwd[:, 0], 'k+', label="VIS")
axs[7].plot(obs.observations[:, 1], fwd[:, 1], 'rx', label="NIR")
axs[7].set_xlabel("Measured BHR [-]")
axs[7].set_ylabel("Predicted BHR [-]")
axs[7].plot ( [0,0.9], [0, 0.9], 'k--', lw=0.5)
axs[7].legend(loc='best')
axs[8].vlines(obs.mask[:, 0], obs.observations[:, 0] - 1.96 * obs.bu[:, 0],
obs.observations[:, 0] + 1.96 * obs.bu[:, 0])
axs[8].plot(obs.mask[:, 0], obs.observations[:, 0], 'o')
axs[9].vlines(obs.mask[:, 0], obs.observations[:, 1] - 1.96 * obs.bu[:, 1],
obs.observations[:, 1] + 1.96 * obs.bu[:, 1])
axs[9].plot(obs.mask[:, 0], obs.observations[:, 1], 'o')
axs[8].set_ylabel("BHR VIS [-]")
axs[9].set_ylabel("BHR NIR [-]")
axs[8].set_xlabel("DoY [d]")
axs[9].set_xlabel("DoY [d]")
for i in xrange(10):
if i != 7:
axs[i].set_xlim(1, 370)
# Hide the right and top spines
axs[i].spines['right'].set_visible(False)
axs[i].spines['top'].set_visible(False)
# Only show ticks on the left and bottom spines
axs[i].yaxis.set_ticks_position('left')
axs[i].xaxis.set_ticks_position('bottom')
fig.figimage(logo, fig.bbox.xmax - 500, fig.bbox.ymax - 250, alpha=.4, zorder=1)#if __name__ == "__main__":
def status_printer(_, total=None, desc=None, ncols=None):
# Prepare IPython progress bar
from ipywidgets import IntProgress, HTML, HBox, Layout, Label
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
description = Label(desc)
description_box = HBox(children=[description])
description_box.layout.min_width = '35%'
description_box.layout.max_width = '35%'
else:
description_box = None
# Prepare status text
ptext = HTML()
# Prepare layout
inner = HBox([pbar, ptext],
layout=Layout(padding='0 0 0 20px'))
# Only way to place text to the right of the bar is to use a container
box_layout = Layout(display='flex',
width='100%')
container = HBox(children=[description_box, inner] if description_box else [inner],
layout=box_layout)
if ncols is not None: # use default style of ipywidgets
# ncols could be 100, "100px", "100%"
ncols = str(ncols) # ipywidgets only accepts string
if ncols[-1].isnumeric():
# if last value is digit, assume the value is digit
ncols += 'px'
pbar.layout.flex = '2'
container.layout.width = ncols
container.layout.display = 'inline-flex'
container.layout.flex_flow = 'row wrap'
from IPython.core.display import display
display(container)
# HTML encoding
try: # Py3
from html import escape
except ImportError: # Py2
from cgi import escape
def print_status(s='', close=False, bar_style=None, desc=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
# n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
try:
container.close()
except AttributeError:
container.visible = False
# Update description
if desc:
description.value = desc
return print_status
def status_printer(file, total=None, desc=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook, tqdm_notebook).status_printer(file)
fp = file
if not getattr(fp, 'flush', False): # pragma: no cover
fp.flush = lambda: None
# Prepare IPython progress bar
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
pbar.description = desc
# Prepare status text
ptext = HTML()
# Only way to place text to the right of the bar is to use a container
container = HBox(children=[pbar, ptext])
display(container)
def print_status(s='', close=False, bar_style=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
container.visible = False
return print_status
def in_progress(seq, msg="Progress: [%(processed)d / %(total)d]",
length=None, close=True):
""" Iterate over sequence, yielding item with progress widget displayed.
This is useful if you need to precess sequence of items with some
time consuming operations
.. note::
This works only in Jupyter Notebook
.. note::
This function requires *ipywidgets* package to be installed
:param seq: sequence to iterate on.
:param str msg: (optional) message template to display.
Following variables could be used in this template:
- processed
- total
- time_total
- time_per_item
:param int length: (optional) if seq is generator, or it is not
possible to apply 'len(seq)' function to 'seq',
then this argument is required and it's value will
be used as total number of items in seq.
Example example::
import time
for i in in_progress(range(10)):
time.sleep(1)
"""
from IPython.display import display
from ipywidgets import IntProgress
import time
if length is None:
length = len(seq)
start_time = time.time()
progress = IntProgress(
value=0, min=0, max=length, description=msg % {
'processed': 0,
'total': length,
'time_total': 0.0,
'time_per_item': 0.0,
'time_remaining': 0.0,
}
)
display(progress)
for i, item in enumerate(seq, 1):
progress.value = i
# i_start_time = time.time()
yield item # Do the job
i_end_time = time.time()
progress.description = msg % {
'processed': i,
'total': length,
'time_total': i_end_time - start_time,
'time_per_item': (i_end_time - start_time) / i,
'time_remaining': ((i_end_time - start_time) / i) * (length - i),
}
if close:
progress.close()
