def templatefile(self, line): """creates a file from template stored as a file """ args = parse_argstring(self.templatefile, line) if not os.path.isfile(args.template): raise FileNotFoundError(args.template) with open(args.template) as f: fread = f.read() output = args.output p = params() if args.format == 'mustache': result = mustache(fread, p, output) elif args.foramt == 'fstring': result = fstring(fread, p, output) else: raise ValueError( f'Unsupported format: {args.format}. Must be: mustache|fstring' ) if output and args.verbose: return display(FileLink(output), Code(result)) elif output: return display(FileLink(output)) else: return display(Code(result))
def make_plots(outdir=None, plotdir=None, outfile=None, verbose=True): """Perform 'make plots' and display links""" import os, sys cmd = 'make plots' if outdir is not None: cmd = cmd + ' OUTDIR=%s' % outdir if plotdir is None: plotdir = '_plots' else: cmd = cmd + ' PLOTDIR=%s' % plotdir if outfile is None: outfile = 'plot_output.txt' cmd = cmd + ' &> %s' % outfile else: cmd = cmd + ' &> %s' % outfile if verbose: print "Making plots... %s" % cmd sys.stdout.flush() status = os.system(cmd) if verbose: local_file = FileLink(outfile) print "Done... Check this file to see output:" display(local_file) index_file = FileLink('%s/_PlotIndex.html' % plotdir) print "View plots created at this link:" display(index_file)
def mcdata2matfile_or_pkl(mcdata, file_name, obs, params = {}, save_mat = True, save_pkl = False): """ Takes an mcdata object and the observables and stores the states, expectations, times, observable labels (str and latex), random seeds, number of trajectories as: { "psis": psis, # shape (ntraj, ntimes, dim_psi) complex128 "expects": expects, # shape (ntraj, ntimes, num_observables) complex128 "times": times, # shape (ntimes) float64 "observable_str": observable_str, # shape (num_observables) string "observable_latex": observable_latex, # shape (num_observables) string "seeds": seeds, # shape (ntraj) int } Additional parameters can be passed using the params argument, as a python dictionary. """ ntraj = mcdata.ntraj assert ntraj >= 1 psis = np.array([ np.vstack([ mcdata.states[jj][tt].data.toarray().astype("complex128").ravel() for tt in range(len(mcdata.times))]) for jj in range(ntraj)]) expects = np.array([np.array(mcdata.expect[jj]).astype("complex128").T for jj in range(ntraj)]) times = mcdata.times observable_str = [str(o) for o in obs] observable_latex = [o._repr_latex_() for o in obs] seeds = mcdata.seeds col_times = mcdata.col_times[0] col_which = mcdata.col_which[0] mdict = { "psis": psis, "expects": expects, "times": times, "observable_str": observable_str, "observable_latex": observable_latex, "seeds": seeds, "col_times": col_times, "col_which": col_which, } mdict.update(params) if save_mat: savemat(file_name, mdict) display(FileLink(file_name+".mat")) if save_pkl: import pickle output = open(file_name + ".pkl", 'wb') pickle.dump( mdict, output,protocol=0) output.close() display(FileLink(file_name+".pkl")) return mdict
def template(self, line, cell=None): """create a template from cell content """ args = parse_argstring(self.template, line) filename = args.filename template = cell.strip() result = mustache(template, params(), filename) if args.verbose: return display(FileLink(filename), Code(result)) else: return display(FileLink(filename))
def add_encryption_output(self, encrypted_text: str, image_path: str, key_path: str): self.encrypted_text_widget = widgets.Text( value=encrypted_text, disabled=True ) display(self.encrypted_text_widget) local_image = FileLink(image_path, result_html_prefix="Click here to download image: ") display(local_image) local_key = FileLink(key_path, result_html_prefix="Click here to download key: ") display(local_key)
def print_todos(): if not need_latex(): html_str = '' for task in bi.__todos__: html_str += "<li>{task}</li>\n".format(task=task) if need_markdown(): return html_str elif need_latex(): # we should write to another file here! # export_todos() return display(FileLink('todos.pdf')) elif need_latex(): export_todos() return display(FileLink('todos.pdf'))
def auto_download(url, dest): if cfg.auto_persistent and os.path.isfile(dest): if IN_IPYTHON: display(FileLink(dest, result_html_prefix="Using local file ")) return dest filename = download(url, suffix=dest) if cfg.auto_persistent: shutil.move(filename, dest) filename = dest if IN_IPYTHON: logger.warning( f"Do not forget attaching \"{dest}\" file with your notebook") display(FileLink(dest, result_html_prefix="Using local file ")) return filename
def save2matfile_or_pkl(data, file_name, obs, params={}, save_mat=True, save_pkl=False): """ Takes an mcdata object and the observables and stores the states, expectations, times, observable labels (str and latex), random seeds, number of trajectories as: { "psis": psis, # shape (ntraj, ntimes, dim_psi) complex128 "expects": expects, # shape (ntraj, ntimes, num_observables) complex128 "times": times, # shape (ntimes) float64 "observable_str": observable_str, # shape (num_observables) string "observable_latex": observable_latex, # shape (num_observables) string "seeds": seeds, # shape (ntraj) int } Additional parameters can be passed using the params argument, as a python dictionary. """ ntraj = data['psis'].shape[0] assert ntraj == data['obsq_expects'].shape[0] assert ntraj >= 1 psis = data['psis'] expects = data['obsq_expects'] times = data['tspan'] observable_str = [str(o) for o in obs] observable_latex = [o._repr_latex_() for o in obs] seeds = data['seeds'] mdict = { "psis": psis, "expects": expects, "times": times, "observable_str": observable_str, "observable_latex": observable_latex, "seeds": seeds, } mdict.update(params) ## other paramters (optional) if save_mat: savemat(file_name, mdict) display(FileLink(file_name + ".mat")) if save_pkl: import pickle output = open(file_name + ".pkl", 'wb') pickle.dump(mdict, output, protocol=0) output.close() display(FileLink(file_name + ".pkl")) return mdict
def notebook_file(filepath): """Display IPython FileLink object in the notebook :param filepath: location of the file """ if filepath: display(FileLink(filepath))
def create_solution(ids, proba, filename='submission_file.csv.zip'): """saves predictions to file and provides a link for downloading """ solution = pandas.DataFrame({'ID': ids}) for name in ['Ghost', 'Electron', 'Muon', 'Pion', 'Kaon', 'Proton']: solution[name] = proba[:, label_class_correspondence[name]] solution.to_csv('{}'.format(filename), index=False, float_format='%.5f', compression="gzip") return FileLink('{}'.format(filename))
def plot_lr_loss(logdir, fn_save=None): fig, ax = plt.subplots(figsize=(8, 8)) with warnings.catch_warnings(): warnings.simplefilter("ignore") train_scalars = get_tensorboard_scalars(logdir + '/train', metrics=['loss', 'lr'], step='batch') tm_loss = get_scatter(train_scalars, 'batch/loss', prefix='train/') tm_lr = get_scatter(train_scalars, 'batch/lr', prefix='train/') ax.plot(np.log10(tm_lr['y']), np.log10(tm_loss['y'])) ax.set_title('LR finder results:') ax.set_xlabel('lg(lr)') ax.set_ylabel('lg(loss)') if fn_save is None: plt.show() else: plt.savefig(fn_save) plt.close(fig) display(FileLink(Path(fn_save))) # return Image(filename=fn_animation) display(HTML(f"<img src='{fn_save}' />"))
def create_solution(best, filename='submission_file.csv'): """saves predictions to file and provides a link for downloading """ df = pd.DataFrame( data=[best], columns=['R', 'pitch', 'y1', 'y2', 'y3', 'z1', 'z2', 'z3']) df.to_csv(filename, index_label=False, index=False) return FileLink('{}'.format(filename))
def export_log_files( cluster_log_delivery_path: PathLike, *, driver_logs: bool = True, event_logs: bool = True, executor_logs: bool = True, init_script_logs: bool = True, after: datetime = None, before: datetime = None, cluster_id: Optional[Union[str, Iterable[str]]] = None, spark_context_id: Optional[Union[str, Iterable[str]]] = None, spark_session_id: Optional[Union[str, Iterable[str]]] = None, spark_app_id: Optional[Union[str, Iterable[str]]] = None, spark_executor_id: Optional[Union[str, Iterable[str]]] = None, ) -> FileLink: # has attr `_repr_html_` that is implicitly displayable in Databricks notebooks find_kwargs = locals() logdirp = cluster_log_delivery_path = convert_to_fuse_path( cluster_log_delivery_path) with TemporaryDirectory( ) as tmpdir: # local to the driver node by necessity tmpdirp = Path(tmpdir) arcdirp = tmpdirp / logdirp.name arcdirp.mkdir() arcfilep = arcdirp.with_suffix(".zip") with ZipFile(arcfilep, "w") as zf: for oldlogp in find_log_files(**find_kwargs): arcname = oldlogp.relative_to(logdirp) newlogp = arcdirp / arcname copyfile(str(oldlogp), str(newlogp)) zf.write(newlogp, arcname) dbfs_arcfilep = Path( "/dbfs/tmp/" ) / f"{arcfilep.stem}_{datetime.now(UTC):%Y%m%dT%H%M%SZ}{arcfilep.suffix}" copyfile(str(arcfilep), str(dbfs_arcfilep)) return FileLink(dbfs_arcfilep)
def get_subm_link(criterion): test_dataset = XRayDataset(f'{data_folder}test.csv', f'{data_folder}', transform=tfms, is_train=False) test_samplers = {'test': sampler.SequentialSampler(test_dataset)} test_dataloaders = { 'test': DataLoader(test_dataset, batch_size=32, sampler=test_samplers['test'], num_workers=8, pin_memory=True) } test_dt_szs = get_dt_szs(test_samplers) # use_gpu = False criterion = criterion t_pdted, t_lbs = predict(dropout_model, 'test', test_dataloaders, test_dt_szs) print(np.bincount(t_pdted)) test_df = pd.read_csv(f'{data_folder}test.csv') test_df.head() test_df['detected'] = pd.Series([ transformed_dataset.idx_to_classes[i] for i in t_pdted ]).astype('category') test_df.drop(['age', 'gender', 'view_position', 'image_name'], axis=1).to_csv('sdir/fst.csv', index=False) return FileLink('./sdir/fst.csv')
def download_link(self, dem): lon, lat = np.meshgrid(np.radians(dem.x), np.radians(dem.y), sparse=True) r = 6371e3 f = np.where(np.isnan(dem.values), 0, dem.values) + r nr, nc = len(dem.y), len(dem.x) self.vertices = np.empty((nr, nc, 3), dtype='float32') self.vertices[:, :, 0] = f * np.cos(lat) * np.cos(lon) self.vertices[:, :, 1] = f * np.cos(lat) * np.sin(lon) self.vertices[:, :, 2] = f * np.sin(lat) self.vertices = self.vertices.reshape(nr * nc, 3) # rotate so that z goes through the middle of the selection r1 = R.from_rotvec(np.radians(np.array([0, 0, -(np.mean(dem.x) + 90)]))) r2 = R.from_rotvec(np.radians(np.array([np.mean(dem.y) - 90, 0, 0]))) self.vertices = r2.apply(r1.apply(self.vertices)) self.vertices[:, 2] -= r self.alt = self.vertices[:, 2].reshape(nr, nc) self.notif.clear_output() np.savez('dem.npz', dem=self.vertices) local_file = FileLink('dem.npz', result_html_prefix="Click here to download: ") with self.notif: display(local_file)
def plot_and_save_corr(df, method='pearson', scale=False, title_postfix=u'', figsize=(16, 6), fn_postfix='all', fn_templ='mc_corr_{method}_{postfix}{datascaled}'): datascaled = '' if scale: datascaled = '_slaced' fn = fn_templ.format(**{'method': method, 'postfix': fn_postfix, 'datascaled': datascaled}) if scale: df = pd.DataFrame(data=preprocessing.scale(df, axis=1), columns=df.columns, index=df.index) df_corr = df.corr(method=method) display(df_corr) df_corr.to_csv(fn + '.csv', encoding='utf8') display(FileLink(fn + '.csv')) m = np.abs(df_corr) print(u"Минимальное значение: {:.4g}".format(m.values.min())) fig, axes = plt.subplots(ncols=2, figsize=figsize) fig.suptitle(u'Корреляционая матрица по ненормированному составу ' u'грунта. {}\n' u'Метод: {}'.format(title_postfix, method), fontsize=14) ax = axes[0] sns.heatmap(m, cmap='hot_r', vmin=0, ax=ax) ax.set_title(u'Диапазон 0-1') ax = axes[1] sns.heatmap(m, cmap='hot_r', ax=ax) ax.set_title(u'Повышенный контраст') ax.text(0.5, -0.1, u"Минимальное значение: {:.4g}".format(m.values.min()), size=12, ha="center", transform=ax.transAxes) fig.savefig(fn + '.png') plt.show() display(FileLink(fn + '.png'))
def on_download_clicked(b): fname = 'logs.txt' hu.save_txt(fname, get_logs(failed_only=True)) bdownload_out.clear_output() with bdownload_out: display(FileLink(fname, result_html_prefix="Download: "))
def display_image(vis, filename='tmp.jpg'): if vis.dtype != np.uint8: imsave(filename, img_as_ubyte(vis)) else: imsave(filename, vis) from IPython.display import FileLink return FileLink(filename)
def on_download_clicked(b): fname = 'results.zip' bdownload_out.clear_output() with bdownload_out: self.rm.to_zip(savedir_base='', fname=fname) bdownload_out.clear_output() with bdownload_out: display('%d exps zipped.' % len(self.rm.exp_list)) display(FileLink(fname, result_html_prefix="Download: "))
def show(lrg, state=None, style=None, fmt=None, save=None, scale="auto", show=True, checkorder=True): # Guess format or fix file extension when saving the image _supported_formats = set(('svg', 'png')) if fmt and fmt not in _supported_formats: fmt = None sfmt = save and save.split(".")[-1] if sfmt not in _supported_formats: sfmt = None fmt = fmt or sfmt if fmt not in _supported_formats: if fmt: print("Unsupported format, revert to default") fmt = _default_image_format img = _get_image(lrg, state, style, fmt, checkorder) if save: if sfmt != fmt: save = "%s.%s" % (save,fmt) print("Saving as %s" % save) mode = "w" if fmt == "svg" else "wb" with open(save, mode) as out: out.write(img) if IN_IPYTHON: display(FileLink(save, result_html_prefix="Saved as ")) if not show: return if not IN_IPYTHON: return img if fmt == "svg": dim = lrg.getDimension() width, height = None, None if scale.endswith("%"): scale = int(scale[:-1])/100 if scale == "auto": width = f"{dim.width}px" if dim.width <= 800 else "100%" elif not isinstance(scale, str): width = f"{int(dim.width*scale)}px" height = f"{int(dim.height*scale)}px" else: width = scale if show == True or show == "img": svg64 = base64.b64encode(img.encode("utf-8")).decode() html = f'<img class="unconfined" width="{width}" src="data:image/svg+xml;base64,{svg64}">' return HTML(html) elif show == "inline": my_width = f'width="{width}"' if width is not None else "" my_height = f'height="{height}"' if height is not None else "" img = _svg_undim(img) img = img.replace("<svg ", f"<svg {my_width} {my_height} viewBox=\"0 0 {dim.width} {dim.height}\" ") else: raise TypeError("Invalid value for show parameter") return show_image(img, is_svg=(fmt=='svg'))
def export_link(self, filepath, output): """Create data URI link and add it to export output area""" self.ctrl.logger.debug('At') output.clear_output() link = FileLink(filepath, result_html_prefix=self.EXPORT_LINK_PROMPT) with output: # noinspection PyTypeChecker display(link)
def create_solution(ids, proba, filename='submission_file.csv.zip'): """saves predictions to file and provides a link for downloading """ solution = pd.DataFrame({'jet_id': ids}) for name in [1, 4, 5, 21]: solution[name] = proba[:, label_class_correspondence[name]] solution.to_csv('{}'.format(filename), index=False, float_format='%.5f', compression="gzip") return FileLink('{}'.format(filename))
def csv_download_link(df, csv_file_name, delete_prompt=True): """Display a download link to load a data frame as csv from within a Jupyter notebook""" import os file_path = os.path.dirname(os.path.abspath(__file__))+"/output"+"/"+csv_file_name df.to_csv(file_path, index=True, sep='\t') from IPython.display import FileLink display(FileLink("./output"+"/"+csv_file_name)) if delete_prompt: a = input('Press enter to delete the file after you have downloaded it.') os.remove(file_path)
def create_solution(ids, proba, names, filename='BaseLine_ICL2018_ANN_Keras.csv'): solution = pd.DataFrame({'ID': ids}) for name in ['Ghost', 'Electron', 'Muon', 'Pion', 'Kaon', 'Proton']: solution[name] = proba[:, np.where(names == name)[0]] solution.to_csv('{}'.format(filename), index=False) return FileLink('{}'.format(filename))
def store_csv(schema, condition=None): file_name = schema.name if not os.path.splitext(file_name)[1] == '.csv': file_name = file_name + '.csv' local_storage_dir = mimic_login.get_local_storage_dir() file_path = os.path.join(local_storage_dir, file_name) loader = _get_table_loader(schema, condition) loader.to_csv(file_path) return FileLink(os.path.relpath(file_path), result_html_prefix='Right-click and save: ')
def df_to_hdf5(file_name, df): if not os.path.splitext(file_name)[1] == '.hdf': file_name = os.path.join(file_name, '.hdf') sp = os.path.splitext(file_name)[0] local_storage_dir = mimic_login.get_local_storage_dir() file_path = os.path.join(local_storage_dir, file_name) store = HdfDataStore(PartialSchema(sp), file_path) store.store(df) return FileLink(os.path.relpath(file_path), result_html_prefix='Right-click and save: ')
def html_open(tmpdir, file): """ Try to open an html file in the OS default web browser. If that fails, provide a link to open it in Jupyter. """ opened = webbrowser.open_new_tab('file://' + os.path.join(tmpdir, file)) if not opened: from IPython.display import FileLink localdir = 'tmp-voyager-' + ''.join( random.choice(string.ascii_letters) for _ in range(6)) shutil.copytree(tmpdir, localdir) return FileLink(os.path.join(localdir, file))
def make_htmls(outfile=None, verbose=False, readme_link=True): """Perform 'make .htmls' and display link.""" import os, sys if outfile is None: outfile = 'htmls_output.txt' cmd = 'make .htmls &> %s' % outfile if verbose: print "Making html documentation files... %s" % cmd sys.stdout.flush() status = os.system(cmd) if verbose: local_file = FileLink(outfile) print "Done... Check this file to see output:" display(local_file) if readme_link: print "See the README.html file for links to input files..." display(FileLink('README.html'))
def save_predictions_to_file(self, fileName=None): if fileName is None: fileName = "predictions." + self.runID + ".h5" outF = open(fileName, 'w') outF.write('image_name,Type_1,Type_2,Type_3\n') for elem in self.prediction: outF.write(elem[0] + ',' + ','.join(elem[1]) + '\n') outF.close() return FileLink(fileName)
def on_download_clicked(b): fname = 'results.zip' bdownload_out.clear_output() self.vars['fname_list'] = get_list_from_str(l_fname_list.value) with bdownload_out: self.rm.to_zip(savedir_base='', fname=fname, fname_list=self.vars['fname_list']) bdownload_out.clear_output() with bdownload_out: display('%d exps zipped.' % len(self.rm.exp_list)) display(FileLink(fname, result_html_prefix="Download: "))