def templatefile(self, line):
"""creates a file from template stored as a file
"""
args = parse_argstring(self.templatefile, line)
if not os.path.isfile(args.template):
raise FileNotFoundError(args.template)
with open(args.template) as f:
fread = f.read()
output = args.output
p = params()
if args.format == 'mustache':
result = mustache(fread, p, output)
elif args.foramt == 'fstring':
result = fstring(fread, p, output)
else:
raise ValueError(
f'Unsupported format: {args.format}. Must be: mustache|fstring'
)
if output and args.verbose:
return display(FileLink(output), Code(result))
elif output:
return display(FileLink(output))
else:
return display(Code(result))
def make_plots(outdir=None, plotdir=None, outfile=None, verbose=True):
"""Perform 'make plots' and display links"""
import os, sys
cmd = 'make plots'
if outdir is not None:
cmd = cmd + ' OUTDIR=%s' % outdir
if plotdir is None:
plotdir = '_plots'
else:
cmd = cmd + ' PLOTDIR=%s' % plotdir
if outfile is None:
outfile = 'plot_output.txt'
cmd = cmd + ' &> %s' % outfile
else:
cmd = cmd + ' &> %s' % outfile
if verbose:
print "Making plots... %s" % cmd
sys.stdout.flush()
status = os.system(cmd)
if verbose:
local_file = FileLink(outfile)
print "Done... Check this file to see output:"
display(local_file)
index_file = FileLink('%s/_PlotIndex.html' % plotdir)
print "View plots created at this link:"
display(index_file)
def mcdata2matfile_or_pkl(mcdata, file_name, obs, params = {}, save_mat = True, save_pkl = False):
"""
Takes an mcdata object and the observables and stores the states, expectations, times,
observable labels (str and latex), random seeds, number of trajectories as:
{
"psis": psis, # shape (ntraj, ntimes, dim_psi) complex128
"expects": expects, # shape (ntraj, ntimes, num_observables) complex128
"times": times, # shape (ntimes) float64
"observable_str": observable_str, # shape (num_observables) string
"observable_latex": observable_latex, # shape (num_observables) string
"seeds": seeds, # shape (ntraj) int
}
Additional parameters can be passed using the params argument, as a python dictionary.
"""
ntraj = mcdata.ntraj
assert ntraj >= 1
psis = np.array([
np.vstack([
mcdata.states[jj][tt].data.toarray().astype("complex128").ravel()
for tt in range(len(mcdata.times))])
for jj in range(ntraj)])
expects = np.array([np.array(mcdata.expect[jj]).astype("complex128").T for jj in range(ntraj)])
times = mcdata.times
observable_str = [str(o) for o in obs]
observable_latex = [o._repr_latex_() for o in obs]
seeds = mcdata.seeds
col_times = mcdata.col_times[0]
col_which = mcdata.col_which[0]
mdict = {
"psis": psis,
"expects": expects,
"times": times,
"observable_str": observable_str,
"observable_latex": observable_latex,
"seeds": seeds,
"col_times": col_times,
"col_which": col_which,
}
mdict.update(params)
if save_mat:
savemat(file_name, mdict)
display(FileLink(file_name+".mat"))
if save_pkl:
import pickle
output = open(file_name + ".pkl", 'wb')
pickle.dump( mdict, output,protocol=0)
output.close()
display(FileLink(file_name+".pkl"))
return mdict
def template(self, line, cell=None):
"""create a template from cell content
"""
args = parse_argstring(self.template, line)
filename = args.filename
template = cell.strip()
result = mustache(template, params(), filename)
if args.verbose:
return display(FileLink(filename), Code(result))
else:
return display(FileLink(filename))
def add_encryption_output(self, encrypted_text: str, image_path: str, key_path: str):
self.encrypted_text_widget = widgets.Text(
value=encrypted_text,
disabled=True
)
display(self.encrypted_text_widget)
local_image = FileLink(image_path, result_html_prefix="Click here to download image: ")
display(local_image)
local_key = FileLink(key_path, result_html_prefix="Click here to download key: ")
display(local_key)
def print_todos():
if not need_latex():
html_str = ''
for task in bi.__todos__:
html_str += "<li>{task}</li>\n".format(task=task)
if need_markdown():
return html_str
elif need_latex():
# we should write to another file here!
# export_todos()
return display(FileLink('todos.pdf'))
elif need_latex():
export_todos()
return display(FileLink('todos.pdf'))
def auto_download(url, dest):
if cfg.auto_persistent and os.path.isfile(dest):
if IN_IPYTHON:
display(FileLink(dest, result_html_prefix="Using local file "))
return dest
filename = download(url, suffix=dest)
if cfg.auto_persistent:
shutil.move(filename, dest)
filename = dest
if IN_IPYTHON:
logger.warning(
f"Do not forget attaching \"{dest}\" file with your notebook")
display(FileLink(dest, result_html_prefix="Using local file "))
return filename
def save2matfile_or_pkl(data,
file_name,
obs,
params={},
save_mat=True,
save_pkl=False):
"""
Takes an mcdata object and the observables and stores the states, expectations, times,
observable labels (str and latex), random seeds, number of trajectories as:
{
"psis": psis, # shape (ntraj, ntimes, dim_psi) complex128
"expects": expects, # shape (ntraj, ntimes, num_observables) complex128
"times": times, # shape (ntimes) float64
"observable_str": observable_str, # shape (num_observables) string
"observable_latex": observable_latex, # shape (num_observables) string
"seeds": seeds, # shape (ntraj) int
}
Additional parameters can be passed using the params argument, as a python dictionary.
"""
ntraj = data['psis'].shape[0]
assert ntraj == data['obsq_expects'].shape[0]
assert ntraj >= 1
psis = data['psis']
expects = data['obsq_expects']
times = data['tspan']
observable_str = [str(o) for o in obs]
observable_latex = [o._repr_latex_() for o in obs]
seeds = data['seeds']
mdict = {
"psis": psis,
"expects": expects,
"times": times,
"observable_str": observable_str,
"observable_latex": observable_latex,
"seeds": seeds,
}
mdict.update(params) ## other paramters (optional)
if save_mat:
savemat(file_name, mdict)
display(FileLink(file_name + ".mat"))
if save_pkl:
import pickle
output = open(file_name + ".pkl", 'wb')
pickle.dump(mdict, output, protocol=0)
output.close()
display(FileLink(file_name + ".pkl"))
return mdict
def notebook_file(filepath):
"""Display IPython FileLink object in the notebook
:param filepath: location of the file
"""
if filepath:
display(FileLink(filepath))
def create_solution(ids, proba, filename='submission_file.csv.zip'):
"""saves predictions to file and provides a link for downloading """
solution = pandas.DataFrame({'ID': ids})
for name in ['Ghost', 'Electron', 'Muon', 'Pion', 'Kaon', 'Proton']:
solution[name] = proba[:, label_class_correspondence[name]]
solution.to_csv('{}'.format(filename), index=False, float_format='%.5f', compression="gzip")
return FileLink('{}'.format(filename))
def plot_lr_loss(logdir, fn_save=None):
fig, ax = plt.subplots(figsize=(8, 8))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
train_scalars = get_tensorboard_scalars(logdir + '/train',
metrics=['loss', 'lr'],
step='batch')
tm_loss = get_scatter(train_scalars, 'batch/loss', prefix='train/')
tm_lr = get_scatter(train_scalars, 'batch/lr', prefix='train/')
ax.plot(np.log10(tm_lr['y']), np.log10(tm_loss['y']))
ax.set_title('LR finder results:')
ax.set_xlabel('lg(lr)')
ax.set_ylabel('lg(loss)')
if fn_save is None:
plt.show()
else:
plt.savefig(fn_save)
plt.close(fig)
display(FileLink(Path(fn_save)))
# return Image(filename=fn_animation)
display(HTML(f"<img src='{fn_save}' />"))
def create_solution(best, filename='submission_file.csv'):
"""saves predictions to file and provides a link for downloading """
df = pd.DataFrame(
data=[best],
columns=['R', 'pitch', 'y1', 'y2', 'y3', 'z1', 'z2', 'z3'])
df.to_csv(filename, index_label=False, index=False)
return FileLink('{}'.format(filename))
def export_log_files(
cluster_log_delivery_path: PathLike,
*,
driver_logs: bool = True,
event_logs: bool = True,
executor_logs: bool = True,
init_script_logs: bool = True,
after: datetime = None,
before: datetime = None,
cluster_id: Optional[Union[str, Iterable[str]]] = None,
spark_context_id: Optional[Union[str, Iterable[str]]] = None,
spark_session_id: Optional[Union[str, Iterable[str]]] = None,
spark_app_id: Optional[Union[str, Iterable[str]]] = None,
spark_executor_id: Optional[Union[str, Iterable[str]]] = None,
) -> FileLink: # has attr `_repr_html_` that is implicitly displayable in Databricks notebooks
find_kwargs = locals()
logdirp = cluster_log_delivery_path = convert_to_fuse_path(
cluster_log_delivery_path)
with TemporaryDirectory(
) as tmpdir: # local to the driver node by necessity
tmpdirp = Path(tmpdir)
arcdirp = tmpdirp / logdirp.name
arcdirp.mkdir()
arcfilep = arcdirp.with_suffix(".zip")
with ZipFile(arcfilep, "w") as zf:
for oldlogp in find_log_files(**find_kwargs):
arcname = oldlogp.relative_to(logdirp)
newlogp = arcdirp / arcname
copyfile(str(oldlogp), str(newlogp))
zf.write(newlogp, arcname)
dbfs_arcfilep = Path(
"/dbfs/tmp/"
) / f"{arcfilep.stem}_{datetime.now(UTC):%Y%m%dT%H%M%SZ}{arcfilep.suffix}"
copyfile(str(arcfilep), str(dbfs_arcfilep))
return FileLink(dbfs_arcfilep)
def get_subm_link(criterion):
test_dataset = XRayDataset(f'{data_folder}test.csv',
f'{data_folder}',
transform=tfms,
is_train=False)
test_samplers = {'test': sampler.SequentialSampler(test_dataset)}
test_dataloaders = {
'test':
DataLoader(test_dataset,
batch_size=32,
sampler=test_samplers['test'],
num_workers=8,
pin_memory=True)
}
test_dt_szs = get_dt_szs(test_samplers)
# use_gpu = False
criterion = criterion
t_pdted, t_lbs = predict(dropout_model, 'test', test_dataloaders,
test_dt_szs)
print(np.bincount(t_pdted))
test_df = pd.read_csv(f'{data_folder}test.csv')
test_df.head()
test_df['detected'] = pd.Series([
transformed_dataset.idx_to_classes[i] for i in t_pdted
]).astype('category')
test_df.drop(['age', 'gender', 'view_position', 'image_name'],
axis=1).to_csv('sdir/fst.csv', index=False)
return FileLink('./sdir/fst.csv')
def download_link(self, dem):
lon, lat = np.meshgrid(np.radians(dem.x),
np.radians(dem.y),
sparse=True)
r = 6371e3
f = np.where(np.isnan(dem.values), 0, dem.values) + r
nr, nc = len(dem.y), len(dem.x)
self.vertices = np.empty((nr, nc, 3), dtype='float32')
self.vertices[:, :, 0] = f * np.cos(lat) * np.cos(lon)
self.vertices[:, :, 1] = f * np.cos(lat) * np.sin(lon)
self.vertices[:, :, 2] = f * np.sin(lat)
self.vertices = self.vertices.reshape(nr * nc, 3)
# rotate so that z goes through the middle of the selection
r1 = R.from_rotvec(np.radians(np.array([0, 0,
-(np.mean(dem.x) + 90)])))
r2 = R.from_rotvec(np.radians(np.array([np.mean(dem.y) - 90, 0, 0])))
self.vertices = r2.apply(r1.apply(self.vertices))
self.vertices[:, 2] -= r
self.alt = self.vertices[:, 2].reshape(nr, nc)
self.notif.clear_output()
np.savez('dem.npz', dem=self.vertices)
local_file = FileLink('dem.npz',
result_html_prefix="Click here to download: ")
with self.notif:
display(local_file)
def plot_and_save_corr(df, method='pearson',
scale=False,
title_postfix=u'',
figsize=(16, 6),
fn_postfix='all',
fn_templ='mc_corr_{method}_{postfix}{datascaled}'):
datascaled = ''
if scale:
datascaled = '_slaced'
fn = fn_templ.format(**{'method': method, 'postfix': fn_postfix,
'datascaled': datascaled})
if scale:
df = pd.DataFrame(data=preprocessing.scale(df, axis=1),
columns=df.columns, index=df.index)
df_corr = df.corr(method=method)
display(df_corr)
df_corr.to_csv(fn + '.csv', encoding='utf8')
display(FileLink(fn + '.csv'))
m = np.abs(df_corr)
print(u"Минимальное значение: {:.4g}".format(m.values.min()))
fig, axes = plt.subplots(ncols=2, figsize=figsize)
fig.suptitle(u'Корреляционая матрица по ненормированному составу '
u'грунта. {}\n'
u'Метод: {}'.format(title_postfix, method),
fontsize=14)
ax = axes[0]
sns.heatmap(m, cmap='hot_r', vmin=0, ax=ax)
ax.set_title(u'Диапазон 0-1')
ax = axes[1]
sns.heatmap(m, cmap='hot_r', ax=ax)
ax.set_title(u'Повышенный контраст')
ax.text(0.5, -0.1,
u"Минимальное значение: {:.4g}".format(m.values.min()),
size=12, ha="center",
transform=ax.transAxes)
fig.savefig(fn + '.png')
plt.show()
display(FileLink(fn + '.png'))
def on_download_clicked(b):
fname = 'logs.txt'
hu.save_txt(fname, get_logs(failed_only=True))
bdownload_out.clear_output()
with bdownload_out:
display(FileLink(fname, result_html_prefix="Download: "))
def display_image(vis, filename='tmp.jpg'):
if vis.dtype != np.uint8:
imsave(filename, img_as_ubyte(vis))
else:
imsave(filename, vis)
from IPython.display import FileLink
return FileLink(filename)
def on_download_clicked(b):
fname = 'results.zip'
bdownload_out.clear_output()
with bdownload_out:
self.rm.to_zip(savedir_base='', fname=fname)
bdownload_out.clear_output()
with bdownload_out:
display('%d exps zipped.' % len(self.rm.exp_list))
display(FileLink(fname, result_html_prefix="Download: "))
def show(lrg, state=None, style=None, fmt=None, save=None,
scale="auto", show=True, checkorder=True):
# Guess format or fix file extension when saving the image
_supported_formats = set(('svg', 'png'))
if fmt and fmt not in _supported_formats:
fmt = None
sfmt = save and save.split(".")[-1]
if sfmt not in _supported_formats: sfmt = None
fmt = fmt or sfmt
if fmt not in _supported_formats:
if fmt: print("Unsupported format, revert to default")
fmt = _default_image_format
img = _get_image(lrg, state, style, fmt, checkorder)
if save:
if sfmt != fmt:
save = "%s.%s" % (save,fmt)
print("Saving as %s" % save)
mode = "w" if fmt == "svg" else "wb"
with open(save, mode) as out:
out.write(img)
if IN_IPYTHON:
display(FileLink(save, result_html_prefix="Saved as "))
if not show:
return
if not IN_IPYTHON:
return img
if fmt == "svg":
dim = lrg.getDimension()
width, height = None, None
if scale.endswith("%"):
scale = int(scale[:-1])/100
if scale == "auto":
width = f"{dim.width}px" if dim.width <= 800 else "100%"
elif not isinstance(scale, str):
width = f"{int(dim.width*scale)}px"
height = f"{int(dim.height*scale)}px"
else:
width = scale
if show == True or show == "img":
svg64 = base64.b64encode(img.encode("utf-8")).decode()
html = f'<img class="unconfined" width="{width}" src="data:image/svg+xml;base64,{svg64}">'
return HTML(html)
elif show == "inline":
my_width = f'width="{width}"' if width is not None else ""
my_height = f'height="{height}"' if height is not None else ""
img = _svg_undim(img)
img = img.replace("<svg ",
f"<svg {my_width} {my_height} viewBox=\"0 0 {dim.width} {dim.height}\" ")
else:
raise TypeError("Invalid value for show parameter")
return show_image(img, is_svg=(fmt=='svg'))
def export_link(self, filepath, output):
"""Create data URI link and add it to export output area"""
self.ctrl.logger.debug('At')
output.clear_output()
link = FileLink(filepath, result_html_prefix=self.EXPORT_LINK_PROMPT)
with output:
# noinspection PyTypeChecker
display(link)
def create_solution(ids, proba, filename='submission_file.csv.zip'):
"""saves predictions to file and provides a link for downloading """
solution = pd.DataFrame({'jet_id': ids})
for name in [1, 4, 5, 21]:
solution[name] = proba[:, label_class_correspondence[name]]
solution.to_csv('{}'.format(filename),
index=False,
float_format='%.5f',
compression="gzip")
return FileLink('{}'.format(filename))
def csv_download_link(df, csv_file_name, delete_prompt=True):
"""Display a download link to load a data frame as csv from within a Jupyter notebook"""
import os
file_path = os.path.dirname(os.path.abspath(__file__))+"/output"+"/"+csv_file_name
df.to_csv(file_path, index=True, sep='\t')
from IPython.display import FileLink
display(FileLink("./output"+"/"+csv_file_name))
if delete_prompt:
a = input('Press enter to delete the file after you have downloaded it.')
os.remove(file_path)
def create_solution(ids,
proba,
names,
filename='BaseLine_ICL2018_ANN_Keras.csv'):
solution = pd.DataFrame({'ID': ids})
for name in ['Ghost', 'Electron', 'Muon', 'Pion', 'Kaon', 'Proton']:
solution[name] = proba[:, np.where(names == name)[0]]
solution.to_csv('{}'.format(filename), index=False)
return FileLink('{}'.format(filename))
def store_csv(schema, condition=None):
file_name = schema.name
if not os.path.splitext(file_name)[1] == '.csv':
file_name = file_name + '.csv'
local_storage_dir = mimic_login.get_local_storage_dir()
file_path = os.path.join(local_storage_dir, file_name)
loader = _get_table_loader(schema, condition)
loader.to_csv(file_path)
return FileLink(os.path.relpath(file_path),
result_html_prefix='Right-click and save: ')
def df_to_hdf5(file_name, df):
if not os.path.splitext(file_name)[1] == '.hdf':
file_name = os.path.join(file_name, '.hdf')
sp = os.path.splitext(file_name)[0]
local_storage_dir = mimic_login.get_local_storage_dir()
file_path = os.path.join(local_storage_dir, file_name)
store = HdfDataStore(PartialSchema(sp), file_path)
store.store(df)
return FileLink(os.path.relpath(file_path),
result_html_prefix='Right-click and save: ')
def html_open(tmpdir, file):
"""
Try to open an html file in the OS default web browser. If that fails,
provide a link to open it in Jupyter.
"""
opened = webbrowser.open_new_tab('file://' + os.path.join(tmpdir, file))
if not opened:
from IPython.display import FileLink
localdir = 'tmp-voyager-' + ''.join(
random.choice(string.ascii_letters) for _ in range(6))
shutil.copytree(tmpdir, localdir)
return FileLink(os.path.join(localdir, file))
def make_htmls(outfile=None, verbose=False, readme_link=True):
"""Perform 'make .htmls' and display link."""
import os, sys
if outfile is None:
outfile = 'htmls_output.txt'
cmd = 'make .htmls &> %s' % outfile
if verbose:
print "Making html documentation files... %s" % cmd
sys.stdout.flush()
status = os.system(cmd)
if verbose:
local_file = FileLink(outfile)
print "Done... Check this file to see output:"
display(local_file)
if readme_link:
print "See the README.html file for links to input files..."
display(FileLink('README.html'))
def save_predictions_to_file(self, fileName=None):
if fileName is None:
fileName = "predictions." + self.runID + ".h5"
outF = open(fileName, 'w')
outF.write('image_name,Type_1,Type_2,Type_3\n')
for elem in self.prediction:
outF.write(elem[0] + ',' + ','.join(elem[1]) + '\n')
outF.close()
return FileLink(fileName)
def on_download_clicked(b):
fname = 'results.zip'
bdownload_out.clear_output()
self.vars['fname_list'] = get_list_from_str(l_fname_list.value)
with bdownload_out:
self.rm.to_zip(savedir_base='', fname=fname,
fname_list=self.vars['fname_list'])
bdownload_out.clear_output()
with bdownload_out:
display('%d exps zipped.' % len(self.rm.exp_list))
display(FileLink(fname, result_html_prefix="Download: "))
