def build_stats_widget(self):
from ipywidgets import FloatProgress, HBox, VBox, HTML, Layout, Button, Box
loss_text = HTML('Loss', width='140px')
self.loss_bar = FloatProgress(min=0.0,
max=1.0,
description='',
height='10px')
loss_widget = HBox([loss_text, self.loss_bar], width='100%')
acc_text = HTML('Accuracy', width='140px')
self.acc_bar = FloatProgress(min=0,
max=1.0,
description='',
height='10px')
acc_widget = HBox([acc_text, self.acc_bar], width='100%')
box_layout = Layout(display='flex',
flex_flow='row',
align_items='stretch',
justify_content='space-around',
border='1px solid #48A7F2',
width='100%')
return Box(children=[acc_widget, loss_widget],
layout=box_layout,
box_style='info')
def __init__(
self,
training_length: Any = None,
update_interval: int = 100,
bar_length: int = 50,
out: Any = sys.stdout,
):
self._training_length = training_length
if training_length is not None:
self._init_status_template()
self._update_interval = update_interval
self._recent_timing: List[Tuple[float, float, float]] = []
self._total_bar = FloatProgress(description='total',
min=0,
max=1,
value=0,
bar_style='info')
self._total_html = HTML()
self._epoch_bar = FloatProgress(description='this epoch',
min=0,
max=1,
value=0,
bar_style='info')
self._epoch_html = HTML()
self._status_html = HTML()
self._widget = VBox([
HBox([self._total_bar, self._total_html]),
HBox([self._epoch_bar, self._epoch_html]), self._status_html
])
def __init__(
self,
training_length=None,
update_interval=100,
):
self._training_length = training_length
self._status_template = None
self._update_interval = update_interval
self._recent_timing = []
self.desc_total = Label("Total:")
self.desc_total.layout.width = "100px"
self.pbar_total = FloatProgress(min=0, max=1.0, bar_style="success")
self.text_total = Label("0%")
self.desc_total.layout.padding = "5px"
self.text_total.layout.padding = "5px"
display(HBox([self.desc_total, self.pbar_total, self.text_total]))
self.desc_epoch = Label("This epoch:")
self.desc_epoch.layout.width = "100px"
self.pbar_epoch = FloatProgress(min=0, max=1.0)
self.text_epoch = Label("0%")
self.desc_epoch.layout.padding = "5px"
self.text_epoch.layout.padding = "5px"
display(HBox([self.desc_epoch, self.pbar_epoch, self.text_epoch]))
self.epoch_report = Label("")
self.time_report = Label("")
self.epoch_report.layout.padding = "5px"
self.time_report.layout.padding = "5px"
display(VBox([self.epoch_report, self.time_report]))
def frech(LINES, mat, n_cores=4, modulo=1):
PQ = np.moveaxis(LINES, 1, 2).copy()
# PQ = PQ.reshape(PQ.shape[0], PQ.shape[1], 1, PQ.shape[2]).copy()
futures = []
progressbar = FloatProgress(min=0, max=100)
display(progressbar)
with ProcessPoolExecutor(n_cores) as executor:
for j in range(num_lines):
# iterr = 0
for i in range(j + 1, num_lines):
if (i - j - 1) % modulo == 0:
# print(i)
futures.append(
executor.submit(disc_frech_wrap,
PQ,
j,
i,
modulo=modulo))
elif num_lines - i < modulo:
futures.append(
executor.submit(disc_frech_wrap,
PQ,
j,
i,
modulo=modulo))
break
# iterr += 1
if j % int(num_lines / 100) == 0:
progressbar.value += 1
progressbar2 = FloatProgress(min=0, max=int(len(futures) / 100))
display(progressbar2)
count = 0
for p in as_completed(futures):
count += 1
if count % int(len(futures) / 100) == 0: progressbar2.value += 1
try:
ma, k, r = p.result()
# ind = 0
# print(ma.shape)
lines_left = num_lines - r
lim = modulo if lines_left > modulo else (num_lines - k - 1)
for l in range(lim):
mat[k, r + l] = ma[l].copy()
mat[r + l, k] = mat[k, r + l]
# ind += 1
except IndexError:
pass
print('Finished processing ', count * modulo, ' distances')
return mat
def __create_exp_progress_box(name, exp_progress, rep_progress, show_full_progress=False):
exp_progress_layout = Layout(display='flex', flex_flow='column', align_items='stretch', width='100%')
exp_progress_bar = HBox([FloatProgress(value=exp_progress, min=.0, max=1., bar_style='info'), Label(name)])
if show_full_progress:
rep_progress_layout = Layout(display='flex', flex_flow='column', align_items='stretch',
align_self='flex-end', width='80%')
items = [FloatProgress(value=p, min=.0, max=1., description=str(i)) for i, p in enumerate(rep_progress)]
rep_progress_box = Box(children=items, layout=rep_progress_layout)
return Box(children=[exp_progress_bar, rep_progress_box], layout=exp_progress_layout)
else:
return exp_progress_bar
def cluster(path, decompositions, dimensions, algos, clustercounts):
runcount = len(decompositions) * len(dimensions) * len(algos) * len(
clustercounts)
f = FloatProgress(min=0, max=runcount)
display(f)
for composition in decompositions:
path_composition = './dist/data/' + path + 'decomposition/' + composition + '/'
for dimension in dimensions:
path_dimension = path_composition + str(dimension) + '/'
P = joblib.load(path_dimension + 'P.pkl')
for algo in algos:
path_algo = path_dimension + algo + '/'
for clustercount in clustercounts:
path_algo_clustercount = path_algo + str(
clustercount) + '/'
clu_pipelines = cluster_pipelines2(clustercount)
pipeline = clu_pipelines[algo]
if hasattr(pipeline, 'predict'):
Y_pred = pipeline.fit(P).predict(P)
else:
Y_pred = pipeline.fit_predict(P)
frames.save(path_algo_clustercount, 'Y_pred', Y_pred)
f.value += 1
def plot_ROC_curves(conv_model):
colors = cycle([
'cyan', 'indigo', 'seagreen', 'yellow', 'blue', 'darkorange', 'red',
'black', 'green', 'brown'
])
#Plot the ROC curves for the training above
for cv, color in zip(range(0, 1), colors):
nbatches = 100
jetImageGenerator2 = JetImageGenerator()
gen = jetImageGenerator2.generator(test=True)
y_predict = []
y_score = []
#Progress bar
#maxval = 100
maxval = nbatches
f = FloatProgress(min=0, max=maxval)
display(f)
for i in range(nbatches):
f.value += 1
#if i%10==0:
# print "Jet",i
Xp, yp = gen.next()
y_predict += [yp]
y_score += [conv_model.predict(Xp)]
y_predict = np.concatenate(y_predict)
y_score = np.concatenate(y_score)
print y_predict
print y_score
# Compute ROC curve and area the curve
fpr, tpr, thresholds = roc_curve(y_predict, y_score)
mean_tpr += interp(mean_fpr, fpr, tpr)
mean_tpr[0] = 0.0
roc_auc = auc(fpr, tpr)
plt.plot(fpr,
tpr,
lw=lw,
color=color,
label='ROC fold %d (area = %0.2f)' % (i, roc_auc))
i += 1
plt.plot([0, 1], [0, 1], linestyle='--', lw=lw, color='k', label='Luck')
mean_tpr /= kfold.get_n_splits(X, encoded_Y)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
plt.figure()
plt.plot(mean_fpr,
mean_tpr,
color='g',
linestyle='--',
label='Mean ROC (area = %0.2f)' % mean_auc,
lw=lw)
plt.xlim([0, 1.0])
plt.ylim([0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.show()
def show(self, a_progress=None, ext='', p_format="{}:{}:{}%"):
"""
进行进度控制显示主方法
:param ext: 可以添加额外的显示文字，str，默认空字符串
:param a_progress: 默认None, 即使用类内部计算的迭代次数进行进度显示
:param p_format: 进度显示格式，默认{}: {}%，即'self._label:round(self._progress / self._total * 100, 2))%'
"""
self.progress = a_progress if a_progress is not None else self.progress + 1
ps = round(self._progress / self._total * 100, 2)
if self._label is not None:
# 如果初始化label没有就只显示ui进度
self.f.write('\r')
self.f.write(p_format.format(self._label, ext, ps))
if ABuEnv.g_is_ipython:
if self.progress_widget is None:
self.progress_widget = FloatProgress(value=0, min=0, max=100)
display(self.progress_widget)
self.progress_widget.value = ps
# 这样会出现余数结束的情况，还是尽量使用上下文管理器控制结束
if self._progress == self._total:
self.f.write('\r')
if self.progress_widget is not None:
self.progress_widget.close()
def predict(self):
'''
Iteratively predict values based on available neighbor data
On each iteration, predictdf attribute is revised
'''
self.predictdf = self.targetdf.copy()
dftest = self.predictdf.copy()
dftest = dftest[pd.isnull(dftest[self.label])]
# Set up progressbar
nullcount = pd.isnull(self.predictdf[self.label]).sum()
if self.progressbar:
maxnullcount = nullcount
f = FloatProgress(min=0, max=maxnullcount)
display(f)
while nullcount > 0:
if ((~pd.isnull(dftest.lead)) &
(~pd.isnull(dftest.lag))).sum() > 0:
dftest = self.predict_once(dftest, lead=True, lag=True)
if (~pd.isnull(dftest.lead)).sum() > 0:
dftest = self.predict_once(dftest, lead=True)
if (~pd.isnull(dftest.lag)).sum() > 0:
dftest = self.predict_once(dftest, lag=True)
nullcount = pd.isnull(self.predictdf[self.label]).sum()
print nullcount
if self.progressbar:
f.value = maxnullcount - nullcount
def change_in_pixels_data(nnet, Xset, Tset, end_pixel_val=10, trials_per_pixel=5, directory='data/', name='data'):
perturbs = ['stuck', 'dead', 'hot']
f = FloatProgress(min=0, max=(end_pixel_val * trials_per_pixel * len(perturbs)))
display(f)
for i, perturb in enumerate(perturbs):
filename = directory + name + '-' + perturb + '.csv'
change = []
for pixels in range(end_pixel_val):
accuracy = []
for trial in range(trials_per_pixel):
Xcopy = change_pixel(Xset, pixels_to_change=pixels+1, pertrub=perturb)
try:
percent = ml.percent_correct(nnet.use(Xcopy)[0], Tset)
except:
percent = ml.percent_correct(ml.batched_use(nnet, Xcopy), Tset)
accuracy.append(percent)
f.value += 1
change.append(accuracy)
pd.DataFrame(change).to_csv(filename, index=False)
try:
natural_per = ml.percent_correct(nnet.use(Xset)[0], Tset)
except:
natural_per = ml.percent_correct(ml.batched_use(nnet, Xset), Tset)
filename = directory + name + '.metadata'
with open(filename, 'w') as f:
print(f'natural: ml.percent_correct(nnet.use(Xset)[0], Tset) = {natural_per}', file=f)
def __init__(self, color=C.BLUE, title="Processing:"):
self._percentage = 0.0
self.visible = False
if is_in_ipynb():
self.set_display_mode("ipynb")
else:
self.set_display_mode("text")
if color == C.BLUE:
color_style = ''
elif color == C.LIGHT_BLUE:
color_style = 'info'
elif color == C.RED:
color_style = 'danger'
elif color == C.LIGHT_RED:
color_style = 'warning'
elif color == C.GREEN:
color_style = 'success'
else:
print('Unavailable color code. Using default "BLUE", instead. ')
color_style = ''
self._pb = FloatProgress(value=0.0,
min=0,
max=100.0,
step=0.1,
description=title,
bar_style=color_style,
orientation='horizontal')
def __init__(self,
iterable=None,
length=None,
*,
label=None,
show_eta=True,
show_percent=None,
show_pos=False,
item_show_func=None,
info_sep=' '):
from traitlets import TraitError
try:
from ipywidgets import FloatProgress
except ImportError:
from IPython.html.widgets.widget_float import FloatProgress
try:
self.backend = FloatProgress(value=0, min=0, step=1)
# max and description are set via properties
except TraitError:
raise RuntimeError('IPython notebook needs to be running')
super().__init__(iterable,
length,
label=label,
show_eta=show_eta,
show_percent=show_percent,
show_pos=show_pos,
item_show_func=item_show_func,
info_sep=info_sep)
self.is_hidden = False
def get_traffic_generator(start, end, projects=('en', )):
global USE_NOTEBOOK
global DATALINE_REGEX
projects = '|'.join(list(projects))
DATALINE_REGEX = re.compile(
r'^(?P<project>{projects:s}) - (?P<hits>\d+)'.format(
projects=projects), re.MULTILINE)
try:
USE_NOTEBOOK
except NameError:
USE_NOTEBOOK = False
else:
pass
if USE_NOTEBOOK:
from ipywidgets import FloatProgress
from IPython.display import display
total_files = (end - start).total_seconds() / 3600
global PROGRESS_BAR
PROGRESS_BAR = FloatProgress(min=0, max=total_files)
display(PROGRESS_BAR)
exploded_dates = map(explode_time, time_range(start, end))
dateurls = map(gen_dateurl, exploded_dates)
data_generators = map(explode_data_files, dateurls)
data_generator = reduce(chain, data_generators)
return data_generator
def extract_text(self, url):
try:
if url.value.startswith('http') and '://' in url.value:
prog = FloatProgress(min=0, max=100, description='Progress')
display(widgets.HTML('<br/>'), prog)
tr0 = time()
site = self.browser.get(url.value, timeout=10)
if site.ok:
prog.value += 50
tr1 = time() - tr0
t0 = time()
cleaner = Cleaner()
cleaner.javascript = True
cleaner.style = True
cleaner.kill_tags = ['header', 'footer']
source_tree = etree.HTML(cleaner.clean_html(site.content))
text = source_tree.itertext()
t1 = time() - t0
self.text = '\n'.join(
[n.strip() for n in text if n.strip()])
prog.value += 50
self.keywords_and_display(prog)
else:
display(
widgets.HTML(
'<div style="font-size: 1.5em; margin-top:1em; margin-bottom:1em">404 - bad URL</div>'
))
else:
self.text = url.value
self.keywords_and_display(False)
except Exception as e:
print 'Error extracting text: %s' % (e)
def pad(cropped, maxes, max):
zf_max, yf_max, xf_max = (maxes[0], maxes[1], maxes[2]
) # The destination size of dimensions
f = FloatProgress(min=0, max=max, description='Padding:')
display(f)
pad_stack = []
for image in cropped:
#Subtracting the destination from the current size to find how much padding is needed
z_diff, y_diff, x_diff = zf_max - image.shape[0], yf_max - image.shape[
1], xf_max - image.shape[2]
z_odd = z_diff % 2
y_odd = y_diff % 2
x_odd = x_diff % 2
# Pad all the dimensions as equally as possible on each side (6 sides in a cube)
if z_odd:
z_pad = (int((z_diff - 1) / 2) + 1, (int((z_diff - 1) / 2)))
else:
z_pad = (int(z_diff / 2), int(z_diff / 2))
if y_odd:
y_pad = (int((y_diff - 1) / 2) + 1, (int((y_diff - 1) / 2)))
else:
y_pad = (int(y_diff / 2), int(y_diff / 2))
if x_odd:
x_pad = (int((x_diff - 1) / 2) + 1, (int((x_diff - 1) / 2)))
else:
x_pad = (int(x_diff / 2), int(x_diff / 2))
pad_stack.append(np.pad(image, [z_pad, y_pad, x_pad], mode='constant'))
f.value += 1
pad_stack = np.array([pad_stack])[0] # Convert to a numpy array
return pad_stack
def create_features_as_matrix(self, samples, show_progress_bar=False):
'''
Creates featres for all the given sample objects.
@return: The created features, as a float numpy matrix (shape: n_samples X n_features).
'''
if show_progress_bar:
from IPython.display import display
from ipywidgets import FloatProgress
progress_bar = FloatProgress(min=0, max=len(samples) - 1)
display(progress_bar)
feature_matrix = np.empty((len(samples), self.n_features()),
dtype=np.float64)
for i, sample in enumerate(samples):
self.create_features_into_array(feature_matrix[i, :], sample)
if show_progress_bar:
progress_bar.value = i
return feature_matrix
def __init__(self,
iterable=None,
length=None,
*,
label=None,
show_eta=True,
show_percent=None,
show_pos=False,
item_show_func=None,
info_sep=' '):
from IPython import get_ipython
try:
from ipywidgets import FloatProgress
except ImportError:
from IPython.html.widgets.widget_float import FloatProgress
ipython = get_ipython()
if not ipython or ipython.__class__.__name__ != 'ZMQInteractiveShell':
raise RuntimeError('IPython notebook needs to be running')
self.backend = FloatProgress(value=0, min=0, step=1)
# max and description are set via properties
super().__init__(iterable,
length,
label=label,
show_eta=show_eta,
show_percent=show_percent,
show_pos=show_pos,
item_show_func=item_show_func,
info_sep=info_sep)
self.is_hidden = False
def read_datalines(self, NSTEPS=0, start_step=-1, select_ckeys=None, max_vector_dim=None, even_NSTEPS=True):
"""Read NSTEPS steps of file, starting from start_step, and store only
the selected ckeys.
INPUT:
NSTEPS -> number of steps to read (default: 0 -> reads all the file)
start_step = -1 -> continue from current step (default)
0 -> go to start step
N -> go to N-th step
select_ckeys -> an array with the column keys you want to read (see all_ckeys for a list)
max_vector_dim -> when reading vectors read only this number of components (None = read all components)
even_NSTEPS -> round the number of steps to an even number (default: True)
OUTPUT:
data -> a dictionary with the selected-column steps
"""
if self._GUI:
progbar = FloatProgress(min=0, max=100)
display(progbar)
start_time = time()
if (NSTEPS == 0):
NSTEPS = self.MAX_NSTEPS
self._set_ckey(select_ckeys, max_vector_dim) # set the ckeys to read
self._initialize_dic(NSTEPS) # allocate dictionary
self.gotostep(start_step) # jump to the starting step
# read NSTEPS of the file
progbar_step = max(100000, int(0.005*NSTEPS))
for step in range(NSTEPS):
line = self.file.readline()
if len(line) == 0: # EOF
print "Warning: reached EOF."
break
values = np.array(line.split())
for key, idx in self.ckey.iteritems(): # save the selected columns
self.data[key][step,:] = np.array(map(float, values[idx]))
if ( (step+1)%progbar_step == 0 ):
if self._GUI:
progbar.value = float(step+1)/NSTEPS*100.;
progbar.description = "{:6.2f}%".format(progbar.value)
else:
print " step = {:9d} - {:6.2f}% completed".format(step+1, float(step+1)/NSTEPS*100.)
if self._GUI:
progbar.close()
# check number of steps read, keep an even number of steps
if (step + 1 < self.NSTEPS):
if (step == 0):
print "WARNING: no step read."
return
else:
print "Warning: less steps read."
self.NSTEPS = step + 1
if even_NSTEPS:
if (NSTEPS%2 == 1):
NSTEPS = NSTEPS - 1
for key, idx in self.ckey.iteritems(): # free memory not used
self.data[key] = self.data[key][:NSTEPS,:]
print " ( %d ) steps read." % (NSTEPS)
self.NSTEPS = NSTEPS
print "DONE. Elapsed time: ", time()-start_time, "seconds"
return self.data
def solve(self, r0):
"""Trace rays through the turbulent grid
Args:
r0 (4xN float): array of N rays, in their initial configuration
"""
f = FloatProgress(min=0,
max=self.ne_grid.shape[0],
description='Progress:')
display(f)
self.r0 = r0 # keep the original
dz = self.z[1] - self.z[0]
DZ = Z1(dz) # matrix to push rays by dz
rt = r0.copy() # iterate to save memory, starting at r0
for i, ne_slice in enumerate(self.ne_grid):
f.value = i
gx, gy = gradient_interpolator(ne_slice, self.x, self.y)
rr1 = deflect_rays(rt, gx, gy, dz=dz)
rt = transform(DZ, rr1)
self.rt = rt
def __init__(self, color=C.BLUE, title="Processing:"):
self._percentage = 0.0
self.visible = False
if is_in_ipynb():
self.set_display_mode("ipynb")
else:
self.set_display_mode("text")
if color == C.BLUE:
color_style = ''
elif color == C.LIGHT_BLUE:
color_style = 'info'
elif color == C.RED:
color_style = 'danger'
elif color == C.LIGHT_RED:
color_style = 'warning'
elif color == C.GREEN:
color_style = 'success'
else:
print('Unavailable color code. Using default "BLUE", instead. ')
color_style = ''
self._pb = FloatProgress(
value=0.0,
min=0,
max=100.0,
step=0.1,
description=title,
bar_style=color_style,
orientation='horizontal',
layout=Layout(padding='0px 0px 0px 10px', width='100%', height='20px')
)
self._pb.style.description_width = 'initial'
self._pb.layout.border='solid 1px'
def make_widget(self, all):
from ipywidgets import FloatProgress, HBox, VBox, HTML
self.elapsed_time = HTML("")
self.bars = {
key: FloatProgress(min=0, max=1, description="")
for key in all
}
self.bar_texts = {key: HTML("") for key in all}
self.bar_labels = {
key:
HTML('<div style="padding: 0px 10px 0px 10px;'
" text-align:left; word-wrap: "
'break-word;">' +
html_escape(key.decode() if isinstance(key, bytes) else key) +
"</div>")
for key in all
}
def keyfunc(kv):
""" Order keys by most numerous, then by string name """
return kv[::-1]
key_order = [
k for k, v in sorted(all.items(), key=keyfunc, reverse=True)
]
self.bar_widgets = VBox([
HBox([self.bar_texts[key], self.bars[key], self.bar_labels[key]])
for key in key_order
])
self.widget.children = (self.elapsed_time, self.bar_widgets)
def multiple_prediction(url_model: str, df_data: pd.DataFrame) -> tuple:
"""Call the API to get the model predictions for the data frame sentences.
Parameters:
url_model (str): API url
df_data (pandas.DataFrame): The dataframe containing intents and sentences
Returns:
np array: The predicted values
np array: The prediction pobabilities for each value
"""
f = FloatProgress(min=0, max=len(df_data))
display(f)
predictions = []
predictions_prob = []
for i in range(len(df_data)):
f.value += 1
sentence = df_data['sentence'][i]
pred = predict(url_model, sentence)
max_val = max(pred, key=pred.get)
predictions.append(max_val)
predictions_prob.append(list(pred.values()))
# prevent API overload
if i % 1000 == 0:
time.sleep(1)
return np.array(predictions), np.array(predictions_prob)
def run_increasing_noise(nnet, Xset, Tset, var_range=(0.001, 0.05),
num_steps=5, trials_per_step=5):
change = []
f = FloatProgress(min=0, max=(num_steps * trials_per_step))
display(f)
for var_step in np.linspace(var_range[0], var_range[1], num_steps):
accuracy = []
for trial in range(trials_per_step):
Xcopy = add_image_noise(Xset, var_step)
try:
percent = ml.percent_correct(nnet.use(Xcopy)[0], Tset)
except:
percent = ml.percent_correct(ml.batched_use(nnet, Xcopy, 1000), Tset)
accuracy.append(percent)
f.value += 1
change.append(accuracy)
change = np.array(change)
x = np.linspace(var_range[0], var_range[1], num_steps)
y = np.mean(change, axis=1)
yerr = np.std(change, axis=1)
return (x, y, yerr)
def download_file(osm_url,dest):
if config.get("general","proxy_https")!="":
urllib2.install_opener(
urllib2.build_opener(
urllib2.ProxyHandler({'https': config.get("general","proxy_https")})
)
)
file_name = "tempdata/"+osm_url.split('/')[-1]
print "downloading: "+osm_url
print "in progress"
u = urllib2.urlopen(osm_url)
f = open(file_name, 'wb')
meta = u.info()
file_size = int(meta.getheaders("Content-Length")[0])
print "Downloading: %s Bytes: %s" % (file_name, file_size)
file_size_dl = 0
block_sz = 8192
progressbar = FloatProgress(min=0, max=100) # instantiate the bar
display(progressbar) # display the bar
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
progressbar.value=file_size_dl * 100. / file_size
f.close()
def increasing_blur_data(nnet, Xset, Tset, var_range=(0.001, 0.05),
num_steps=5, trials_per_step=5, directory='data/', name='data'):
change = []
f = FloatProgress(min=0, max=(num_steps * trials_per_step))
display(f)
for var_step in np.linspace(var_range[0], var_range[1], num_steps):
filename = directory + name + '-' + str(var_step) + '.csv'
accuracy = []
for trial in range(trials_per_step):
Xcopy = add_image_blur(Xset, var_step)
try:
percent = ml.percent_correct(nnet.use(Xcopy)[0], Tset)
except:
percent = ml.percent_correct(ml.batched_use(nnet, Xcopy), Tset)
accuracy.append(percent)
f.value += 1
change.append(accuracy)
pd.DataFrame(change).to_csv(filename, index=False)
try:
natural_per = ml.percent_correct(nnet.use(Xset)[0], Tset)
except:
natural_per = ml.percent_correct(ml.batched_use(nnet, Xset), Tset)
filename = directory + name + '.metadata'
with open(filename, 'w') as f:
print(f'natural: ml.percent_correct(nnet.use(Xset)[0], Tset) = {natural_per}', file=f)
def add_bar(self, job):
self.bars[job] = FloatProgress(
value=0.0,
min=0.0,
max=100.0,
description="Job: %04d Stage: %04d" % (job, 0),
bar_style="info",
orientation="horizontal",
style=self.style,
)
self.bars[job].add_class("db-bar")
self.labels[job] = Label(
value="",
description="Code:",
disabled=False,
layout=Layout(width="800px", height="100%", margin="0 0 0 5px"),
)
self.labels[job].add_class("db-label")
progressbar = HBox([self.bars[job], self.labels[job]])
self.progressbars.children = self.progressbars.children + (
progressbar, )
if not self.progressbar_showing:
self.progressbar_showing = True
display(self.indicator)
def download_and_extract(data_dir, print_progress=True):
"""Download and extract the tarball from Alex's website."""
if not os.path.exists(data_dir):
os.makedirs(data_dir)
if os.path.exists(os.path.join(data_dir, 'cifar-10-batches-bin')):
print('cifar dataset already downloaded')
return
filename = CIFAR_DOWNLOAD_URL.split('/')[-1]
filepath = os.path.join(data_dir, filename)
if not os.path.exists(filepath):
f = FloatProgress(min=0, max=100)
display(f)
sys.stdout.write('\r>> Downloading %s ' % (filename))
def _progress(count, block_size, total_size):
if print_progress:
f.value = 100.0 * count * block_size / total_size
filepath, _ = urllib.request.urlretrieve(CIFAR_DOWNLOAD_URL, filepath,
_progress)
print()
statinfo = os.stat(filepath)
print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
tarfile.open(filepath, 'r:gz').extractall(data_dir)
def __init__(self, total):
super().__init__()
from ipywidgets import FloatProgress
from IPython.display import display
self.progress = FloatProgress(min=0, max=total)
display(self.progress)
def __init__(self, a_pid):
"""通过进程pid初始化ui组件"""
self.progress_widget = FloatProgress(value=0, min=0, max=100)
self.text_widget = Text('pid={} begin work'.format(a_pid))
# 通过box容器都放到一个里面
self.progress_box = Box([self.text_widget, self.progress_widget])
display(self.progress_box)
def _wrapper(self, *args, **kwargs): # pylint: disable=too-many-locals
result = func(self, *args, **kwargs)
if isinstance(result, rockets.RequestTask) and in_notebook():
from ipywidgets import FloatProgress, Label, HBox, VBox, Button
from IPython.display import display
progress = FloatProgress(min=0, max=1, value=0)
label = Label(value='')
button = Button(description='Cancel')
box = VBox([label, HBox([progress, button])])
display(box)
def _on_cancel(value): # pylint: disable=unused-argument
result.cancel()
def _on_progress(value):
progress.value = value.amount
label.value = value.operation
def _on_done(task): # pylint: disable=unused-argument
box.close()
button.on_click(_on_cancel)
result.add_progress_callback(_on_progress)
result.add_done_callback(_on_done)
return result
