def tile_image(full_size_path, tile_size=400):
p = IntProgress(description='Tiling', max=1)
display(p)
root_dir = os.path.dirname(full_size_path)
name, ext = os.path.splitext(os.path.basename(full_size_path))
tile_dir = os.path.join(root_dir, '{0}-{1}x{1}'.format(name, tile_size))
if not os.path.exists(tile_dir):
os.mkdir(tile_dir)
tpl = os.path.join(tile_dir, '{name}-{i:02}-{j:02}{ext}')
full_size = skimage.io.imread(full_size_path)
X, Y = full_size.shape[:2]
total_tiles = (X // tile_size) * (Y // tile_size)
p.max = total_tiles
print("Creating %i tiles in %s" % (total_tiles, tile_dir))
for i in range(X // tile_size):
for j in range(Y // tile_size):
tile = full_size[
i * tile_size: (i+1) * tile_size,
j * tile_size: (j+1) * tile_size,
:
]
fname = tpl.format(**locals())
p.value += 1
skimage.io.imsave(fname, tile)
p.value = p.max
class PBinJ(object):
""" initialize multiple progress bars for tracking nested stages of fitting routine
"""
def __init__(self, n=1, value=0, status='{}', color='b', width='60%', height='22px'):
self.displayed = False
self.style_bar(n=n, value=value, status=status, color=color, width=width, height=height)
def style_bar(self, n=1, value=0, status='{}', color='b', width='60%', height='22px'):
colordict = {'g': 'success', 'b': '', 'r': 'danger', 'y': 'warning', 'c': 'info'}
bar_style = colordict[color]
self.bar = IntProgress(min=0, max=n, value=value, bar_style=bar_style)
self.status = status
self.bar.bar_style = bar_style
self.bar.width = width
self.bar.height = height
def reset_bar(self, color=False):
self.update(value=0)
def update(self, value=None, status=None):
if not self.displayed:
display(self.bar)
self.displayed=True
if status is not None:
if hasattr(status, '__iter__') and not isinstance(status, str):
status = self.status.format(*status)
else:
status = self.status.format(status)
self.bar.description = status
if value is not None:
self.bar.value = value+1
def clear(self):
self.bar.close()
self.displayed = False
class PBinJ(object):
""" initialize multiple progress bars for tracking nested stages of fitting routine
"""
def __init__(self, n=1, value=0, status="{}", color="r", width="50%", height="25px"):
self.displayed = False
self.style_bar(n=n, value=value, status=status, color=color, width=width, height=height)
def style_bar(self, n=1, value=0, status="{}", color="r", width="50%", height="25px"):
colordict = {"g": "#16a085", "b": "#4168B7", "r": "#e74c3c", "y": "#f39c12"}
self.bar = IntProgress(min=0, max=n, value=value)
self.status = status
self.bar.color = colordict[color]
self.bar.width = width
self.bar.height = height
def reset_bar(self):
self.update(value=0)
def update(self, value=None, status=None):
if not self.displayed:
display(self.bar)
self.displayed = True
if status is not None:
if hasattr(status, "__iter__"):
status = self.status.format(*status)
else:
status = self.status.format(status)
self.bar.description = status
if value is not None:
self.bar.value = value + 1
def clear(self):
self.bar.close()
def style_bar(self, n=1, value=0, status="{}", color="r", width="50%", height="25px"):
colordict = {"g": "#16a085", "b": "#4168B7", "r": "#e74c3c", "y": "#f39c12"}
self.bar = IntProgress(min=0, max=n, value=value)
self.status = status
self.bar.color = colordict[color]
self.bar.width = width
self.bar.height = height
def style_bar(self, n=1, value=0, status='{}', color='b', width='60%', height='22px'):
colordict = {'g': 'success', 'b': '', 'r': 'danger', 'y': 'warning', 'c': 'info'}
bar_style = colordict[color]
self.bar = IntProgress(min=0, max=n, value=value, bar_style=bar_style)
self.status = status
self.bar.bar_style = bar_style
self.bar.width = width
self.bar.height = height
def in_progress(seq, msg="Progress: [%(processed)d / %(total)d]", length=None):
""" Iterate over sequence, yielding item with progress widget displayed.
This is useful if you need to precess sequence of items with some
time consuming operations
.. note::
This works only in Jupyter Notebook
.. note::
This function requires *ipywidgets* package to be installed
:param seq: sequence to iterate on.
:param str msg: (optional) message template to display.
available to use 'processed' and 'total' integer vars,
where 'processed' is number of items processed and
'total' is total number of items in seq.
:param int length: (optional) if seq is generator, or it is not
possible to apply 'len(seq)' function to 'seq',
then this argument is required and it's value will
be used as total number of items in seq.
Example example::
import time
for i in in_progress(range(10)):
time.sleep(1)
"""
from IPython.display import display
from ipywidgets import IntProgress
if length is None:
length = len(seq)
progress = IntProgress(value=0, min=0, max=length,
description=msg % {'processed': 0,
'total': length})
display(progress)
for i, item in enumerate(seq, 1):
progress.value = i
progress.description = msg % {'processed': i, 'total': length}
yield item
progress.close()
def download_original():
p = IntProgress(max=1, description="Downloading")
display(p)
if os.path.exists(full_size_file):
print("Already have %s" % full_size_file)
p.value = p.max
else:
r = requests.get(url, stream=True)
content_length = r.headers.get('content-length', int(1e8))
print("Downloading %s" % url)
p.max = content_length
r.raise_for_status()
with open(full_size_file, 'wb') as f:
for chunk in r.iter_content(chunk_size=8096):
p.value += len(chunk)
f.write(chunk)
p.value = p.max
def log_progress(sequence, every=None, size=None, name='Items'):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(name=name,
index=str(index or '?'))
def setup_play_level(level, on_update, interval=1000, min=1, max=8):
play = Play(interval=interval, min=min, max=max, step=1)
progress = IntProgress(min=min, max=max, step=1)
link((play, 'value'), (progress, 'value'))
play.observe(on_update, 'value')
canvas_dimensions = level.get_canvas_dimensions()
layout = Layout(width=f'{canvas_dimensions[0]}px')
return play, progress, layout
def getHistoricData(symbols, delay=0.5, **options):
'''
get data from Yahoo finance and return pandas dataframe
Will get OHLCV data frame if sinle symbol is provided.
If many symbols are provided, it will return a wide panel
Parameters
------------
symbols : str or list
Yahoo finanance symbol or a list of symbols
sDate : tuple (optional)
start date (y,m,d)
adjust : bool
T/[F] adjust data based on adj_close
Returns
---------
DataFrame, multi-index
'''
assert isinstance(
symbols,
(list, str)), 'Input must be a string symbol or a list of symbols'
if isinstance(symbols, str):
return getSymbolData(symbols, **options)
else:
data = {}
print('Downloading data:')
p = IntProgress(min=0, max=len(symbols) - 1)
display(p)
for idx, symbol in enumerate(symbols):
p.value = idx
try:
data[symbol] = getSymbolData(symbol, verbose=False, **options)
except Exception as e:
print('Failed downloading ' + symbol)
print(e)
sleep(delay)
return pd.concat(data, axis=1, names=['symbol', 'ohlcv'])
def __init__(self, size=100, name='Progress'):
self.size = size
self.name = name
self.t0 = dt.datetime.now()
progress = IntProgress(min=0, max=size, value=0, bar_style='info')
label0 = HTML(value='%s: %d / %d' % (self.name, 0, self.size))
label1 = HTML(value='Time: --')
self.box = VBox(children=[VBox([label0, label1]), progress])
display(self.box)
def log_progress(sequence, every=None, size=None):
is_iterator = False
start_tic = time.time()
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
# pretty human readable time diff
if False:
import dateutil.relativedelta
attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']
delta = dateutil.relativedelta.relativedelta(seconds=time.time()-start_tic)
elapsed = " ".join(['%d %s' % (getattr(delta, attr),
getattr(delta, attr) > 1 and attr or attr[:-1]) for attr in attrs if
getattr(delta, attr)])+" %d usec"%((tdiff-int(tdiff))*1000000)
label.value = u'{index} : {elapsed}'.format(index=index or '?', elapsed=elapsed)
else:
# simple time in sec
label.value = u'{index} : {elapsed:0.2f}s'.format(index=index or '?', elapsed=time.time()-start_tic)
def define_int_progress_bar():
""" define progress bar """
# pylint: disable=line-too-long
return IntProgress(value=0,
min=0,
max=10,
step=1,
description='Loading:',
bar_style='success',
orientation='horizontal',
position='top')
def perceptron_pass1(lines, classifier):
p = IntProgress(max=len(lines))
display(p)
random.shuffle(lines)
for line in lines:
label, feature = get_label_feature(line)
value = get_value(classifier, feature)
if label * value > 0:
# prediction matches
pass
else:
# update each dimension of the classifier
for word in feature:
if None == classifier.get(word):
classifier[word] = label * feature[word]
else:
classifier[word] += label * feature[word]
p.value += 1
p.description = "{}%".format(round(100 * (p.value / p.max), 1))
return classifier
def fix_names2(players):
## Progress bar
progress = IntProgress(min=0, max=len(players), value=0)
display(progress)
arr = []
for i, x in enumerate(players['name']):
surname = ''
tmp = x.split(' ')
if len(tmp) > 1:
surname = " ".join(tmp[1:])
name = tmp[0][0] + '.'
arr.append(surname + " " + name)
else:
arr.append(np.nan)
## Progress fill
if (i % 10 == 0):
progress.value = i
players['name'] = arr
print("PLAYERS' NAMES FIXED\n")
def __init__(self,
gen,
total=None,
display=True,
leave=True,
parent=None,
auto_update=True):
self.progress, self.text = IntProgress(
min=0, max=len(gen) if total is None else total), HTML()
self.box = HBox([self.progress, self.text])
super().__init__(gen, total, display, leave, parent, auto_update)
def create_retirement_stat(df):
players = list(set(df['player1']) or set(df['player2']))
print("CREATING STAT RETIREMENT...")
arr1 = [0 for x in range(len(df))]
arr2 = [0 for x in range(len(df))]
## Progress bar
progress = IntProgress(min=0, max=len(players), value=0)
display(progress)
for k, player in enumerate(players):
subset = df[(df['player1'] == player) | (df['player2'] == player)]
for i in range(1, len(subset)):
if len(subset) > 1:
player1 = list(subset['player1'])
status = list(subset['status'])
player2 = list(subset['player2'])
fl = list(subset['player1_win'])
if (player1[i] == player):
if ((status[i - 1] == 'Retired')
and (player1[i - 1] == player)
and (fl[i - 1] == 0)):
arr1[subset.index[i]] = 1
if ((status[i - 1] == 'Retired')
and (player2[i - 1] == player)
and (fl[i - 1] == 1)):
arr1[subset.index[i]] = 1
else:
if ((status[i - 1] == 'Retired')
and (player1[i - 1] == player)
and (fl[i - 1] == 0)):
arr2[subset.index[i]] = 1
if ((status[i - 1] == 'Retired')
and (player2[i - 1] == player)
and (fl[i - 1] == 1)):
arr2[subset.index[i]] = 1
## Progress fill
if (k % 10 == 0):
progress.value = k
df['1st_match_since_retirement_player1'] = arr1
df['1st_match_since_retirement_player2'] = arr2
print("STAT RETIREMENT CREATED\n")
def get_press_series(spliter, color):
paddings = 4
white_width = 17 + 2 * paddings
black_width = 16 + 2 * paddings
height = 106
width = 884
print('Start extracting keypress series ...')
print(f' White width: {white_width}px')
print(f' Black width: {black_width}px')
for name in spliter:
black_coor = None
N = y_org[name].shape[0]
for p in X_path[name]:
img = cv2.imread(p)
black_coor = get_black_boundaries(img)
if len(black_coor) == 36:
break
bar = IntProgress(max=88 * N)
display(bar)
for k in range(88):
last = -1
for i in range(N):
if y_org[name][i][k] > 0:
if last == -1:
last = i
if y_org[name][i][k] <= 0 or i == N - 1:
if last != -1:
if k in black_mask:
add_series(name, 'black', last, i - 1, k, paddings,
black_coor)
else:
add_series(name, 'white', last, i - 1, k, paddings)
last = -1
bar.value += 1
bar.close()
print(f'{name} set loading finished ...')
print(' Pressed white keys: ' + str(len(X_series[name]['white'])))
print(' Pressed black keys: ' + str(len(X_series[name]['black'])))
def progress_bar(sequence, every=None, size=None, name='Progress'):
'''Source: https://github.com/kuk/log-progress. Makes progress bar during iterating through sequence. Example:
from time import sleep
for i in progress_bar([1,2,3,4],every=1):
sleep(1)'''
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
class lstm_data_batch:
def __init__(self,
type='train',
color='white',
NCHW=True,
shuffle=True,
need_bar=True,
max_num=-1):
self.type = type
self.color = color
self.NCHW = NCHW
self.need_bar = need_bar
if max_num == -1:
self.max_num = len(X_series[type][color])
else:
self.max_num = max_num
self.order = np.arange(self.max_num)
if shuffle:
random.shuffle(self.order)
if need_bar:
self.bar = IntProgress(max=self.max_num)
display(self.bar)
def __iter__(self):
self.index = 0
return self
def __next__(self):
if self.index >= self.max_num:
if self.need_bar:
self.bar.close()
raise StopIteration
ind = self.order[self.index]
X_return = X_series[self.type][self.color][ind]
y_return = y_series[self.type][self.color][ind]
if self.NCHW:
X_return = np.transpose(X_return, (0, 3, 1, 2))
self.index += 1
if self.need_bar:
self.bar.value += 1
return (np.array(X_return), np.array(y_return))
def __init__(self,
type='train',
color='white',
NCHW=True,
shuffle=True,
need_bar=True,
max_num=-1):
self.type = type
self.color = color
self.NCHW = NCHW
self.need_bar = need_bar
if max_num == -1:
self.max_num = len(X_series[type][color])
else:
self.max_num = max_num
self.order = np.arange(self.max_num)
if shuffle:
random.shuffle(self.order)
if need_bar:
self.bar = IntProgress(max=self.max_num)
display(self.bar)
def fix_names1(players):
print("FIXING PLAYERS' NAMES...")
progress = IntProgress(min=0, max=len(players), value=0)
display(progress)
arr = []
for i, x in enumerate(players['name']):
surname = ''
tmp = x.split(' ')
if len(tmp) > 1:
surname = tmp[-1]
name = tmp[0][0] + '.'
arr.append(surname + " " + name)
else:
arr.append(np.nan)
## Progress fill
if (i % 10 == 0):
progress.value = i
players['name'] = arr
def __init__(self, goal):
self.bar = IntProgress(min=0, max=goal, value=0)
self.label = HTML()
box = HBox(children=[self.bar, self.label])
self.goal = goal
self.value = 0
self.template = '{{0}} / {goal}'.format(goal=goal)
self.set_value(0)
display(box)
def __init__(self, label_msg_html="Progress: ", displaybox=True):
super(_SimpleProgressBar_JupyterImpl, self).__init__()
self.progress_widget = IntProgress(min=0, max=100, value=0)
self.label = HTML("<span style=\"padding-right: 1em;\">" +
label_msg_html + "</span>")
self.labelval = HTML()
self.innerhbox = HBox(
children=[self.label, self.progress_widget, self.labelval])
self.addinfo = HTML()
self.vbox = VBox(children=[self.innerhbox, self.addinfo])
if displaybox:
display(self.vbox)
def progress_bar(sequence, every=None, size=None, name='Parameters'):
# c&p from https://github.com/kuk/log-progress
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
IPython.display.display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(name=name,
index=index)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name, index=index, size=size)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = f"{name}: {str(index or '?')}"
def agregar_periodo_vulnerabilidad(modelos_df, serie_vulnerabilidad, inicio,
fin):
inicio = pd.to_datetime(inicio, yearfirst=True)
fin = pd.to_datetime(fin, yearfirst=True)
fechas = pd.date_range(inicio, fin)
resultados = []
modelos = []
asegura_archivos_covid_disponibles(fechas)
f = IntProgress(min=0, max=len(fechas) - 1) # instantiate the bar
display(f) # display the bar
for count, fecha in enumerate(fechas):
covid_municipal = tabla_covid_indicadores_municipales(
fecha.strftime("%y%m%d"))
covid_municipal = agregar_tasas_municipales(covid_municipal)
caracteristicas = caracteristicas_modelos_municipios(covid_municipal)
pls = ajustar_pls_letalidad(covid_municipal, caracteristicas)
df = calificar_municipios_letalidad_formato_largo(covid_municipal,
pls,
caracteristicas,
modelo='PLS',
dia_ajuste=fecha)
resultados.append(df)
modelo = pd.DataFrame({
'caracteristica': caracteristicas,
'coef': pls.coef_
})
modelo['dia_ajuste'] = fecha
modelos.append(modelo)
f.value = count
resultados_df = pd.concat(resultados, ignore_index=True)
nuevos_modelos_df = pd.concat(modelos, ignore_index=True)
serie_vulnerabilidad = serie_vulnerabilidad.append(resultados_df)
modelos_df = modelos_df.append(nuevos_modelos_df)
return modelos_df, serie_vulnerabilidad
def log_progress(sequence, every=None, size=None, name='Items'):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = int(size / 200) # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{name}: {index} / ?'.format(
name=name,
index=index
)
else:
progress.value = index
label.value = u'{name}: {index} / {size}'.format(
name=name,
index=index,
size=size
)
yield record
except Exception:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = "{name}: {index}".format(
name=name,
index=str(index or '?')
)
def extract_features(model, filenames, path_to_images, display_bar=True):
'''
Extracts output from model on list of filenames.
Parameters
----------
model : Keras model.
filenames : list
List of filenames to process.
path_to_images : path to folder with images
Returns
-------
features : ndarray
Extracted features.
'''
if display_bar:
bar = IntProgress(value=1,
min=1,
max=len(filenames),
step=1,
description='Initializing...')
display(bar)
features = np.zeros((len(filenames), 4096), dtype=np.float)
for i in range(len(filenames)):
img_path = os.path.join(path_to_images, filenames[i])
img = image.load_img(img_path, target_size=(224, 224))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
features[i] = feature_extractor.predict(x).squeeze()
if display_bar:
bar.value = i + 1
bar.description = '[{:>{tab}} / {}]'.format(i + 1,
bar.max,
tab=len(str(bar.max)))
if display_bar:
bar.bar_style = 'success'
return features
def log_progress(sequence, every=None, size=None):
"""Widget based progress bar for Jupyter (IPython Notebook)
Author: Kukushkin Alexander
Source: https://github.com/alexanderkuk/log-progress
"""
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, 'sequence is iterator, set every'
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = 'info'
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = '{index} / ?'.format(index=index)
else:
progress.value = index
label.value = u'{index} / {size}'.format(
index=index,
size=size
)
yield record
except:
progress.bar_style = 'danger'
raise
else:
progress.bar_style = 'success'
progress.value = index
label.value = str(index or '?')
def dotrain(model,
sess,
loss,
aug,
Xtr,
ytr,
nepochs=50,
batchsize=100,
title=None):
opt = tf.train.AdadeltaOptimizer(learning_rate=1, rho=0.95,
epsilon=1e-7).minimize(loss)
# noinspection PyInterpreter
progress = IntProgress(value=0,
min=0,
max=nepochs,
description='Training:')
display(progress)
#model.training()
costs = []
for e in range(nepochs):
batchcosts = []
for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True):
if aug is not None:
Xb, yb = aug.augbatch_train(Xb, yb)[0]
sess.run([opt, loss], feed_dict={model.X: Xb, model.y: yb})
# model.zero_grad_parameters()
# cost = model.accumulate_gradients(Xb, yb, crit)
# opt.update_parameters(model)
batchcosts.append(cost)
costs.append(np.mean(batchcosts))
progress.value = e + 1
liveplot(plotcost, costs, title)
return costs
def cliRunSync(module,
node=None,
parameters=None,
delete_temporary_files=True,
update_display=True):
"""Run CLI module. If ipywidgets are installed then it reports progress.
"""
try:
from ipywidgets import IntProgress
from IPython.display import display
# Asynchronous run, with progerss reporting using widget
node = slicer.cli.run(module,
node=node,
parameters=parameters,
wait_for_completion=False,
delete_temporary_files=delete_temporary_files,
update_display=update_display)
import time
progress = IntProgress()
display(progress) # display progress bar
while node.IsBusy():
progress.value = node.GetProgress()
slicer.app.processEvents()
time.sleep(.3)
progress.layout.display = 'none' # hide progress bar
except ImportError:
# No widgets, therefore no progress reporting - do just a simpe synchronous CLI run
node = slicer.cli.runSync(
module,
node=node,
parameters=parameters,
wait_for_completion=False,
delete_temporary_files=delete_temporary_files,
update_display=update_display)
return node
class data_batch:
def __init__(self, size, NCHW=True, concatenate=False):
if size != 'single' and size != 'bundle':
raise ValueError("Expected 'single' or 'bundle'")
if concatenate:
raise NotImplementedError
self.len = len(X_path_list)
self.bundle = (size == 'bundle')
self.NCHW = NCHW
self.concatenate = concatenate
def __len__(self):
return self.len
def __iter__(self):
self.index = 0
self.bar = IntProgress(max=self.len)
display(self.bar)
return self
def __next__(self):
if self.index >= self.len:
self.bar.close()
raise StopIteration
else:
img_path = X_path_list[self.index]
self.index += 1
self.bar.value += 1
img = cv2.imread(img_path)
white_keys = get_white_keys(
img, (bundle_paddings if self.bundle else single_paddings))
black_keys = get_black_keys(
img, black_coor,
(bundle_paddings if self.bundle else single_paddings))
if self.NCHW:
white_keys = np.transpose(white_keys, (0, 3, 1, 2))
black_keys = np.transpose(black_keys, (0, 3, 1, 2))
return white_keys, black_keys
def __init__(self,
type='train',
size='single',
color='white',
batch_size=64,
need_velocity=True,
NCHW=True,
shuffle=True,
max_num=-1):
self.size = size
self.type = type
self.color = color
self.batch_size = batch_size
self.NCHW = NCHW
self.max_num = max_num
self.pressed = []
self.unpressed = []
self.num_pressed = 0
self.num_unpressed = 0
self.need_velocity = need_velocity
for i, x in enumerate(y[color][type]):
if x > 0:
self.pressed.append(i)
self.num_pressed += 1
else:
self.unpressed.append(i)
self.num_unpressed += 1
if shuffle:
random.shuffle(self.pressed)
random.shuffle(self.unpressed)
if self.max_num == -1:
self.max_num = len(self.unpressed)
self.iter_num = len(self.unpressed) * 2
else:
self.max_num = self.max_num // 2
self.iter_num = max_num
self.bar = IntProgress(max=self.iter_num)
display(self.bar)
def load_final_forests(folder_path):
f = IntProgress(min=0,
max=5,
description='Loading... ',
bar_style='success')
display(f)
final_forests = []
for i in range(5):
forest_path = os.path.join(folder_path, str(i), "forest_training.pkl")
clf = pickle.load(open(forest_path, "rb"))
final_forests.append(clf)
f.value += 1
return final_forests
def __init__(self, desc, interval=.01):
from IPython.display import display
from ipywidgets import HBox, IntProgress, HTML
self._total = 0
self.value = 0
self._last_update = 0
self.interval = interval
prog = self.progress = IntProgress(description=desc, value=0, max=0)
text = HTML()
def _update_text(change):
text.value = f"{prog.value}/{prog.max}"
_update_text(None)
prog.observe(_update_text, names=['value', 'max'])
display(HBox([prog, text]))
def __init__(self, *args, **kargs):
# Ipython gives warnings when using widgets about the API potentially changing
with warnings.catch_warnings():
warnings.simplefilter("ignore")
try:
from ipywidgets import IntProgress, HTML, HBox
except ImportError: # Support IPython < 4.0
from IPython.html.widgets import IntProgress, HTML, HBox
super(ProgressIPy, self).__init__(*args, **kargs)
self.prog = IntProgress(max=self.length)
self._label = HTML()
self._box = HBox((self.prog, self._label))
def get_cities(wikiurl=True,
population=True,
coordinates=True,
alias=True,
start=0,
end=100000):
BASE_URL = "https://de.wikipedia.org"
url = BASE_URL + "/wiki/Liste_der_Gro%C3%9F-_und_Mittelst%C3%A4dte_in_Deutschland"
soup = BeautifulSoup(requests.get(url).content, "html.parser")
tables = soup.find_all("table", {"class": "wikitable"})
trs = []
for table in tables:
new_trs = table.find_all("tr")
if len(trs) < len(new_trs):
trs = new_trs
cities = {}
api_params = []
if coordinates:
api_params.append(api_props["coordinates"])
if alias:
api_params.append(api_props["alias"])
f = IntProgress(min=0, max=len(trs[start:end]) - 1) # instantiate the bar
display(f)
for tr in trs[start:end]:
try:
f.value += 1
tds = tr.find_all("td")
city_name = tds[1].find("a").get("title")
data = {}
wikiname = tds[1].find("a").get("href").rsplit("/", 1)[-1]
if wikiurl:
data["wikiurl"] = BASE_URL + tds[1].find("a").get("href")
if population:
data["population"] = int(tds[-2].text.replace(".", ""))
if len(api_params) > 0:
citydata = wiki_api(title=wikiname, props=api_params)
data = {**data, **citydata}
data["names"] = data.get("names", []) + [wikiname]
cities[city_name] = data
except:
pass
return cities
def getJNBProgressBar():
'''
return a progress bar for jupyter-lab / jupyter notebook, to be used with the runCommand method
requires ipywidgets
'''
from ipywidgets import IntProgress
from IPython.display import display
pb = IntProgress(min=0, max=100)
display(pb)
def callback(value):
pb.value = value
return callback
def __init__(self, maxs):
"""Initialization.
Positional argument:
maxs - List containing the max value of each progress bar
"""
self.__bars = [
HBox([
IntProgress(0, 0, max, description='{:.2f}%'.format(0)),
Label("{} / {}".format(0, max))
]) for max in maxs
]
display(VBox(self.__bars))
def __init__(self, name: str, size: int, increments: int = 0):
print(name, flush=True)
from ipywidgets import HBox, IntProgress, Label
from IPython.display import display, clear_output
self.size = size
self._percent = Label("")
self._bar = IntProgress(min=0, max=size)
self._rate = Label("")
self._duration = Label("")
ui = HBox([
self._percent, self._bar,
Label(sizeof_fmt(size)), self._rate, self._duration
])
display(ui)
logger.debug(f"'{type(self)}' displayed IPython widget '{ui}'.")
def dotrain(model, crit, aug, Xtr, ytr, nepochs=50, batchsize=100, title=None):
opt = df.AdaDelta(rho=0.95, eps=1e-7, lr=1)
progress = IntProgress(value=0, min=0, max=nepochs, description="Training:")
display(progress)
model.training()
costs = []
for e in range(nepochs):
batchcosts = []
for Xb, yb in batched(batchsize, Xtr, ytr, shuf=True):
if aug is not None:
Xb, yb = aug.augbatch_train(Xb, yb)
model.zero_grad_parameters()
cost = model.accumulate_gradients(Xb, yb, crit)
opt.update_parameters(model)
batchcosts.append(cost)
costs.append(np.mean(batchcosts))
progress.value = e + 1
liveplot(plotcost, costs, title)
return costs
def __init__(self,
max_it,
name='Progress:',
est_cycle_time=None,
dummy=False):
if debug:
print("new style progress bar")
self._dummy = dummy
if self._dummy:
return
self.starttime = time.time()
self.start_eta_time = self.starttime
self.max_it = max_it
self.name = name
self.progr = 0
#check for old (finished) progressbar with the same name
for p in pb_list:
if p[0] == self.name:
p[1].close()
p[2].close()
self.pb = IntProgress(
value=0,
min=0,
max=self.max_it,
description=self.name,
layout={"width": "95%"},
)
self.pi = HTML(
#value = "(0/%i) <br>✈ -?- <br>🕐 --:--:-- (estimated)<br>✚ 00:00:00 (elapsed) <br>⚊ --:--:-- (remaining)"% (self.max_it),
value=
"<table style='width:100%%'><tr><td>%s (%i/%i) </td><td>✈ %s </td><td>🕐 %s (estimated)</td><td>✚ %s (elapsed) </td><td>⚊ %s (remaining)</td></tr></table>"
% ("", 0, self.max_it,
"-?-" if est_cycle_time == None else time.strftime(
'%Y-%m-%d (%a) %H:%M:%S',
time.localtime(time.time() +
est_cycle_time * self.max_it)),
"--:--:--" if est_cycle_time == None else hourformat(
est_cycle_time * self.max_it), "00:00:00",
"--:--:--" if est_cycle_time == None else hourformat(
est_cycle_time * self.max_it)), )
display(self.pi)
display(self.pb)
def log_progress(sequence, name, every=None, size=None, who="reads"):
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
is_iterator = False
if size is None:
try:
size = len(sequence)
except TypeError:
is_iterator = True
if size is not None:
if every is None:
if size <= 200:
every = 1
else:
every = size / 200 # every 0.5%
else:
assert every is not None, "sequence is iterator, set every"
if is_iterator:
progress = IntProgress(min=0, max=1, value=1)
progress.bar_style = "info"
else:
progress = IntProgress(min=0, max=size, value=0)
label = HTML()
box = VBox(children=[label, progress])
display(box)
index = 0
try:
for index, record in enumerate(sequence, 1):
if index == 1 or index % every == 0:
if is_iterator:
label.value = name + ": " + "{index} {who} / ?".format(index=index, who=who)
else:
progress.value = index
label.value = u"{name}: {index} / {size}".format(name=name, index=index, size=size)
yield record
except:
progress.bar_style = "danger"
raise
else:
progress.bar_style = "success"
progress.value = index
label.value = str(name + ": " + str(index) + " " + str(who) or "?")
def progress_bar(job):
'''Display a status bar showing how many tasks are completed'''
status=get_status(job)
f = IntProgress(min=0, max=status['Total'], bar_style='success')
f.value = status['Total'] - status['Pending'] - status['Running']
f.description = "{:1.0f} tasks left ".format(status['Pending']+status['Running'])
display(f)
while f.value < status['Total']:
status=get_status(job)
f.value = status['Total'] - status['Pending'] - status['Running']
f.description = "{:1.0f} tasks left ".format(status['Pending']+status['Running'])
if status['Failed'] > 0:
f.bar_style='warning'
sleep(1)
def int_progress(min_, max_):
prog = IntProgress(min=min_, max=max_)
display(prog)
for i in linspace(min_, max_, 25):
time.sleep(0.1)
prog.value = i
def in_progress(seq, msg="Progress: [%(processed)d / %(total)d]",
length=None, close=True):
""" Iterate over sequence, yielding item with progress widget displayed.
This is useful if you need to precess sequence of items with some
time consuming operations
.. note::
This works only in Jupyter Notebook
.. note::
This function requires *ipywidgets* package to be installed
:param seq: sequence to iterate on.
:param str msg: (optional) message template to display.
Following variables could be used in this template:
- processed
- total
- time_total
- time_per_item
:param int length: (optional) if seq is generator, or it is not
possible to apply 'len(seq)' function to 'seq',
then this argument is required and it's value will
be used as total number of items in seq.
Example example::
import time
for i in in_progress(range(10)):
time.sleep(1)
"""
from IPython.display import display
from ipywidgets import IntProgress
import time
if length is None:
length = len(seq)
start_time = time.time()
progress = IntProgress(
value=0, min=0, max=length, description=msg % {
'processed': 0,
'total': length,
'time_total': 0.0,
'time_per_item': 0.0,
'time_remaining': 0.0,
}
)
display(progress)
for i, item in enumerate(seq, 1):
progress.value = i
# i_start_time = time.time()
yield item # Do the job
i_end_time = time.time()
progress.description = msg % {
'processed': i,
'total': length,
'time_total': i_end_time - start_time,
'time_per_item': (i_end_time - start_time) / i,
'time_remaining': ((i_end_time - start_time) / i) * (length - i),
}
if close:
progress.close()
def status_printer(_, total=None, desc=None, ncols=None):
# Prepare IPython progress bar
from ipywidgets import IntProgress, HTML, HBox, Layout, Label
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
description = Label(desc)
description_box = HBox(children=[description])
description_box.layout.min_width = '35%'
description_box.layout.max_width = '35%'
else:
description_box = None
# Prepare status text
ptext = HTML()
# Prepare layout
inner = HBox([pbar, ptext],
layout=Layout(padding='0 0 0 20px'))
# Only way to place text to the right of the bar is to use a container
box_layout = Layout(display='flex',
width='100%')
container = HBox(children=[description_box, inner] if description_box else [inner],
layout=box_layout)
if ncols is not None: # use default style of ipywidgets
# ncols could be 100, "100px", "100%"
ncols = str(ncols) # ipywidgets only accepts string
if ncols[-1].isnumeric():
# if last value is digit, assume the value is digit
ncols += 'px'
pbar.layout.flex = '2'
container.layout.width = ncols
container.layout.display = 'inline-flex'
container.layout.flex_flow = 'row wrap'
from IPython.core.display import display
display(container)
# HTML encoding
try: # Py3
from html import escape
except ImportError: # Py2
from cgi import escape
def print_status(s='', close=False, bar_style=None, desc=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
# n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
try:
container.close()
except AttributeError:
container.visible = False
# Update description
if desc:
description.value = desc
return print_status
def display_multi_progress(results, objectlist, sleep=1):
prog = IntProgress(min=0, max=len(list(objectlist))-1)
display(prog)
while not results.ready():
prog.value = results.progress
time.sleep(sleep)
def animator(progbar,
count,
tot_string=False,
linenum=False,
terminal=False,
init=False,
length=False,
**kwargs
):
"""
Animates progress bar in unique position in terminal
Multiple progress bars not supported in jupyter yet.
"""
# if ipython?
welcome_message = kwargs.pop('welcome_message', '')
if welcome_message:
welcome_message = welcome_message.replace('Interrogating corpus ... \n', '')
welcome_message = welcome_message.replace('Concordancing corpus ... \n', '')
welcome_message = welcome_message.replace('\n', '<br>').replace(' ' * 17, ' ' * 17)
else:
welcome_message = ''
if init:
from traitlets import TraitError
try:
from ipywidgets import IntProgress, HTML, VBox
from IPython.display import display
progress = IntProgress(min=0, max=length, value=1)
using_notebook = True
progress.bar_style = 'info'
label = HTML()
label.font_family = 'monospace'
gblabel = HTML()
gblabel.font_family = 'monospace'
box = VBox(children=[label, progress, gblabel])
display(box)
return box
except TraitError:
pass
except ImportError:
pass
if not init:
try:
from ipywidgets.widgets.widget_box import FlexBox
if isinstance(progbar, FlexBox):
label, progress, goodbye = progbar.children
progress.value = count
if count == length:
progress.bar_style = 'success'
else:
label.value = '%s\nInterrogating: %s ...' % (welcome_message, tot_string)
return
except:
pass
# add startnum
start_at = kwargs.get('startnum', 0)
if start_at is None:
start_at = 0.0
denominator = kwargs.get('denom', 1)
if kwargs.get('note'):
if count is None:
perc_done = 0.0
else:
perc_done = (count * 100.0 / float(length)) / float(denominator)
kwargs['note'].progvar.set(start_at + perc_done)
kwargs['root'].update()
return
if init:
from corpkit.textprogressbar import TextProgressBar
return TextProgressBar(length, dirname=tot_string)
# this try is for sublime text nosetests, which don't take terminal object
try:
with terminal.location(0, terminal.height - (linenum + 1)):
if tot_string:
progbar.animate(count, tot_string)
else:
progbar.animate(count)
# typeerror for nose
except:
if tot_string:
progbar.animate(count, tot_string)
else:
progbar.animate(count)
def eoldas_inversion ( fluxnet_site, year, n_years, green_leaves, gamma_lai,
n_tries=5 ):
f = IntProgress(min=0, max=2*n_tries + 1)
f.value = 1
display(f)
prior_set = []
for train_year in xrange ( n_years ):
this_year = (year - n_years - 1) + train_year
retval_s, state, obs = tip_inversion( this_year, fluxnet_site, green_leaves=green_leaves,
n_tries=2 )
prior_set.append ( retval_s )
mu_priors = np.array( [ state.__unpack_from_dict (p['real_map']) for p in prior_set ] )
cov_mtx = np.zeros (( 322, 322))
for p in prior_set:
cov_mtx = cov_mtx + p['hessian'].todense()
prior = Prior ( mu_priors.mean(axis=0), cov_mtx )
mu = state.operators['Prior'].mu
cinv = state.operators['Prior'].inv_cov
c = np.array(np.sqrt(np.linalg.inv (cinv.todense()).diagonal())).squeeze()
post_sd = np.sqrt(np.array(retval_s['post_cov'].todense()).squeeze())
post_sd_single = np.where(post_sd > c, c, post_sd)
retval, state, obs = regularised_tip_inversion( year, fluxnet_site, [1e-3, 0, 0.1, 1e-3, 0, 0.1, gamma_lai ],
x0=retval_s['real_map'], green_leaves=green_leaves,
n_tries=n_tries, progressbar=f)
mu = state.operators['Prior'].mu
cinv = state.operators['Prior'].inv_cov
c = np.array(np.sqrt(np.linalg.inv (cinv.todense()).diagonal())).squeeze()
post_sd = np.sqrt(np.array(retval_s['post_cov'].todense()).squeeze())
post_sd = np.where(post_sd > c, c, post_sd)
fig, axs = plt.subplots(nrows=5, ncols=2, figsize=(14, 12))
axs = axs.flatten()
fig.suptitle("%s (%d)" % (fluxnet_site, year), fontsize=18)
params = ['omega_vis', 'd_vis', 'a_vis', 'omega_nir', 'd_nir', 'a_nir', 'lai']
post_sd = np.sqrt(np.array(retval['post_cov'].todense()).squeeze())
post_sd = np.where(post_sd > c, c, post_sd)
for i, p in enumerate(tip_params):
#axs[i].axhspan(mu[(i*46):((i+1)*46)][0]+c[(i*46):((i+1)*46)][0],
# mu[(i*46):((i+1)*46)][0] - c[(i * 46):((i + 1) * 46)][0], color="0.9" )
axs[i].fill_between ( state.state_grid, retval['real_map'][params[i]] - post_sd[(i*46):((i+1)*46)],
retval['real_map'][params[i]] + post_sd[(i*46):((i+1)*46)], lw=0.8, color="0.8")
axs[i].vlines ( state.state_grid, retval_s['real_map'][params[i]] - post_sd_single[(i*46):((i+1)*46)],
retval_s['real_map'][params[i]] + post_sd_single[(i*46):((i+1)*46)], lw=0.8,
colors="0.1", alpha=0.5)
axs[i].plot(state.state_grid, retval['real_map'][params[i]], 'o-', mfc="none")
axs[i].plot(state.state_grid, retval_s['real_map'][params[i]], '--')
if i in [ 1, 4, 6]:
axs[i].set_ylim(0, 6)
else:
axs[i].set_ylim(0, 1)
axs[i].set_ylabel( tip_params[i] )
fwd = np.array(obs.fwd_modelled_obs)
axs[7].plot(obs.observations[:, 0], fwd[:, 0], 'k+', label="VIS")
axs[7].plot(obs.observations[:, 1], fwd[:, 1], 'rx', label="NIR")
axs[7].set_xlabel("Measured BHR [-]")
axs[7].set_ylabel("Predicted BHR [-]")
axs[7].plot ( [0,0.9], [0, 0.9], 'k--', lw=0.5)
axs[7].legend(loc='best')
axs[8].vlines(obs.mask[:, 0], obs.observations[:, 0] - 1.96 * obs.bu[:, 0],
obs.observations[:, 0] + 1.96 * obs.bu[:, 0])
axs[8].plot(obs.mask[:, 0], obs.observations[:, 0], 'o')
axs[9].vlines(obs.mask[:, 0], obs.observations[:, 1] - 1.96 * obs.bu[:, 1],
obs.observations[:, 1] + 1.96 * obs.bu[:, 1])
axs[9].plot(obs.mask[:, 0], obs.observations[:, 1], 'o')
axs[8].set_ylabel("BHR VIS [-]")
axs[9].set_ylabel("BHR NIR [-]")
axs[8].set_xlabel("DoY [d]")
axs[9].set_xlabel("DoY [d]")
for i in xrange(10):
if i != 7:
axs[i].set_xlim(1, 370)
# Hide the right and top spines
axs[i].spines['right'].set_visible(False)
axs[i].spines['top'].set_visible(False)
# Only show ticks on the left and bottom spines
axs[i].yaxis.set_ticks_position('left')
axs[i].xaxis.set_ticks_position('bottom')
fig.figimage(logo, fig.bbox.xmax - 500, fig.bbox.ymax - 250, alpha=.4, zorder=1)#if __name__ == "__main__":
def status_printer(file, total=None, desc=None):
"""
Manage the printing of an IPython/Jupyter Notebook progress bar widget.
"""
# Fallback to text bar if there's no total
# DEPRECATED: replaced with an 'info' style bar
# if not total:
# return super(tqdm_notebook, tqdm_notebook).status_printer(file)
fp = file
if not getattr(fp, 'flush', False): # pragma: no cover
fp.flush = lambda: None
# Prepare IPython progress bar
if total:
pbar = IntProgress(min=0, max=total)
else: # No total? Show info style bar with no progress tqdm status
pbar = IntProgress(min=0, max=1)
pbar.value = 1
pbar.bar_style = 'info'
if desc:
pbar.description = desc
# Prepare status text
ptext = HTML()
# Only way to place text to the right of the bar is to use a container
container = HBox(children=[pbar, ptext])
display(container)
def print_status(s='', close=False, bar_style=None):
# Note: contrary to native tqdm, s='' does NOT clear bar
# goal is to keep all infos if error happens so user knows
# at which iteration the loop failed.
# Clear previous output (really necessary?)
# clear_output(wait=1)
# Get current iteration value from format_meter string
if total:
n = None
if s:
npos = s.find(r'/|/') # cause we use bar_format=r'{n}|...'
# Check that n can be found in s (else n > total)
if npos >= 0:
n = int(s[:npos]) # get n from string
s = s[npos + 3:] # remove from string
# Update bar with current n value
if n is not None:
pbar.value = n
# Print stats
if s: # never clear the bar (signal: s='')
s = s.replace('||', '') # remove inesthetical pipes
s = escape(s) # html escape special characters (like '?')
ptext.value = s
# Change bar style
if bar_style:
# Hack-ish way to avoid the danger bar_style being overriden by
# success because the bar gets closed after the error...
if not (pbar.bar_style == 'danger' and bar_style == 'success'):
pbar.bar_style = bar_style
# Special signal to close the bar
if close and pbar.bar_style != 'danger': # hide only if no error
container.visible = False
return print_status
def eoldas_inversion ( fluxnet_site, year, green_leaves, gamma_lai,
albedo_unc_avg, albedo_unc_good,
n_tries=10 ):
f = IntProgress(min=0, max=2*n_tries + 1)
f.value = 1
display(f)
# convert uncertainties from percent
albedo_unc = [ albedo_unc_good*0.01, albedo_unc_avg*0.01 ]
retval_s, state, obs = tip_inversion( year, fluxnet_site, albedo_unc=albedo_unc,
green_leaves=green_leaves,
n_tries=n_tries, progressbar=f )
mu = state.operators['Prior'].mu
cinv = state.operators['Prior'].inv_cov
c = np.array(np.sqrt(np.linalg.inv (cinv.todense()).diagonal())).squeeze()
post_sd = np.sqrt(np.array(retval_s['post_cov'].todense()).squeeze())
post_sd_single = np.where(post_sd > c, c, post_sd)
retval, state, obs = regularised_tip_inversion( year, fluxnet_site, [1e-3, 0, 0.1, 1e-3, 0, 0.1, gamma_lai ],
albedo_unc=albedo_unc,
x0=retval_s['real_map'], green_leaves=green_leaves,
n_tries=n_tries, progressbar=f)
mu = state.operators['Prior'].mu
cinv = state.operators['Prior'].inv_cov
c = np.array(np.sqrt(np.linalg.inv (cinv.todense()).diagonal())).squeeze()
post_sd = np.sqrt(np.array(retval_s['post_cov'].todense()).squeeze())
post_sd = np.where(post_sd > c, c, post_sd)
fig, axs = plt.subplots(nrows=5, ncols=2, figsize=(14, 12))
axs = axs.flatten()
fig.suptitle("%s (%d)" % (fluxnet_site, year), fontsize=18)
params = ['omega_vis', 'd_vis', 'a_vis', 'omega_nir', 'd_nir', 'a_nir', 'lai']
post_sd = np.sqrt(np.array(retval['post_cov'].todense()).squeeze())
post_sd = np.where(post_sd > c, c, post_sd)
for i, p in enumerate(tip_params):
#axs[i].axhspan(mu[(i*46):((i+1)*46)][0]+c[(i*46):((i+1)*46)][0],
# mu[(i*46):((i+1)*46)][0] - c[(i * 46):((i + 1) * 46)][0], color="0.9" )
axs[i].fill_between ( state.state_grid, retval['real_map'][params[i]] - post_sd[(i*46):((i+1)*46)],
retval['real_map'][params[i]] + post_sd[(i*46):((i+1)*46)], lw=0.8, color="0.8")
axs[i].vlines ( state.state_grid, retval_s['real_map'][params[i]] - post_sd_single[(i*46):((i+1)*46)],
retval_s['real_map'][params[i]] + post_sd_single[(i*46):((i+1)*46)], lw=0.8,
colors="0.1", alpha=0.5)
axs[i].plot(state.state_grid, retval['real_map'][params[i]], 'o-', mfc="none")
axs[i].plot(state.state_grid, retval_s['real_map'][params[i]], '--')
if i in [ 1, 4, 6]:
axs[i].set_ylim(0, 6)
else:
axs[i].set_ylim(0, 1)
axs[i].set_ylabel( tip_params[i] )
fwd = np.array(obs.fwd_modelled_obs)
axs[7].plot(obs.observations[:, 0], fwd[:, 0], 'k+', label="VIS")
axs[7].plot(obs.observations[:, 1], fwd[:, 1], 'rx', label="NIR")
axs[7].set_xlabel("Measured BHR [-]")
axs[7].set_ylabel("Predicted BHR [-]")
axs[7].plot ( [0,0.9], [0, 0.9], 'k--', lw=0.5)
axs[7].legend(loc='best')
axs[8].vlines(obs.mask[:, 0], obs.observations[:, 0] - 1.96 * obs.bu[:, 0],
obs.observations[:, 0] + 1.96 * obs.bu[:, 0])
axs[8].plot(obs.mask[:, 0], obs.observations[:, 0], 'o')
axs[9].vlines(obs.mask[:, 0], obs.observations[:, 1] - 1.96 * obs.bu[:, 1],
obs.observations[:, 1] + 1.96 * obs.bu[:, 1])
axs[9].plot(obs.mask[:, 0], obs.observations[:, 1], 'o')
axs[8].set_ylabel("BHR VIS [-]")
axs[9].set_ylabel("BHR NIR [-]")
axs[8].set_xlabel("DoY [d]")
axs[9].set_xlabel("DoY [d]")
for i in xrange(10):
if i != 7:
axs[i].set_xlim(1, 370)
# Hide the right and top spines
axs[i].spines['right'].set_visible(False)
axs[i].spines['top'].set_visible(False)
# Only show ticks on the left and bottom spines
axs[i].yaxis.set_ticks_position('left')
axs[i].xaxis.set_ticks_position('bottom')
fig.figimage(logo, fig.bbox.xmax - 500, fig.bbox.ymax - 250, alpha=.4, zorder=1)
plt.savefig("regularised_model_%s_%04d_%02dpcnt_%02dpcnt.pdf" % (fluxnet_site, year,
int(albedo_unc[0]*100.), int(albedo_unc[1]*100.)),
dpi=300, bbox_inches="tight")
