def createPyramidLevel(self, resolution=0, subdiv=(1, 2, 2), quiet=False): """ Add a level in a multi-level pyramid. Provided this function because TeraStitcher does not have enough control over the sampling strategy for imaris files """ # find all of the imaris datasets under the specified resolution group self._subdiv = subdiv datasetnames = list() resin = 'ResolutionLevel ' + str(resolution) resout = 'ResolutionLevel ' + str(resolution + 1) prf = '/DataSet/' + resin self._file_object[prf].visit(datasetnames.append) tt = [ type(self._file_object[prf + '/' + x]) == h5py._hl.dataset.Dataset for x in datasetnames ] res = list(compress(datasetnames, tt)) # Now we need to find the ones ending in '/Data' tt = [x.endswith('/Data') for x in res] res = list(compress(res, tt)) outpaths = ['/DataSet/' + resout + '/' + x for x in res] inpaths = [prf + '/' + x for x in res] pbar = ProgressBar() for idx in range(len(inpaths)): if not quiet: print(inpaths[idx]) pbar.register() self._subdivide(self._file_object, inpaths[idx], outpaths[idx]) if not quiet: pbar.unregister()
def show_progress(arr, msg: str = None, nthreads: int = 1): from dask.diagnostics import ProgressBar if msg is not None: logger.info(msg) pbar = ProgressBar() pbar.register() res = controlled_compute(arr, nthreads) pbar.unregister() return res
def main(): paths = list(Path(args.dir).rglob("*.txt")) pbar = ProgressBar() pbar.register() a_bag = db.from_sequence(paths, npartitions=mp.cpu_count()) a_bag = a_bag.map(lambda a_path: parse_path(a_path)) frame_data = a_bag.compute() pbar.unregister() frame = pd.DataFrame(frame_data) frame.to_pickle(args.out)
def test_register(capsys): try: p = ProgressBar() p.register() assert Callback.active get_threaded(dsk, "e") check_bar_completed(capsys) p.unregister() assert not Callback.active finally: Callback.active.clear()
def test_register(capsys): try: p = ProgressBar() p.register() assert _globals['callbacks'] get(dsk, 'e') check_bar_completed(capsys) p.unregister() assert not _globals['callbacks'] finally: _globals['callbacks'].clear()
def test_register(capsys): try: p = ProgressBar() p.register() assert Callback.active get_threaded(dsk, 'e') check_bar_completed(capsys) p.unregister() assert not Callback.active finally: Callback.active.clear()
def test_register(capsys): try: p = ProgressBar() p.register() assert _globals['callbacks'] get_threaded(dsk, 'e') check_bar_completed(capsys) p.unregister() assert not _globals['callbacks'] finally: _globals['callbacks'].clear()
def test_register(capsys): try: p = ProgressBar() p.register() assert _globals['callbacks'] get(dsk, 'e') out, err = capsys.readouterr() bar, percent, time = [i.strip() for i in out.split('\r')[-1].split('|')] assert bar == "[########################################]" assert percent == "100% Completed" p.unregister() assert not _globals['callbacks'] finally: _globals['callbacks'].clear()
def show_progress(arr, msg: str = None, nthreads: int = 1): """ Performs computation with Dask and shows progress bar. Args: arr: msg: message to log, default None nthreads: number of threads to use for computation, default 1 Returns: Result of computation. """ from dask.diagnostics import ProgressBar if msg is not None: logger.info(msg) pbar = ProgressBar() pbar.register() res = controlled_compute(arr, nthreads) pbar.unregister() return res
def Movie( da, odir, varname=None, framedim="time", moviename="movie", clim=None, cmap=None, bgcolor=np.array([1, 1, 1]) * 0.3, framewidth=1280, frameheight=720, dpi=100, lon=None, lat=None, dask=True, delete=True, ffmpeg=True, plot_style="simple", norm=mpl.colors.Normalize(), progbar=False, ): # Set defaults: if not ffmpeg and delete: raise RuntimeError("raw picture deletion makes only \ sense if ffmpeg conversion is enabled") if not isinstance(da, xr.DataArray): raise RuntimeError("input has to be an xarray DataStructure, instead\ is " + str(type(da))) if not os.path.exists(odir): os.makedirs(odir) # Infer defaults from data if clim is None: print("clim will be inferred from data, this can take very long...") clim = [da.min(), da.max()] if cmap is None: cmap = plt.cm.viridis if plot_style in ["map"]: if None in [lon, lat]: raise RuntimeError("map plotting requires lon and lat") else: lons = np.array(da[lon].data) lats = np.array(da[lat].data) if len(lons.shape) != 2: lons, lats = np.meshgrid(lons, lats) time = np.array(da["time"].data) else: lons = None lats = None time = None # Annnd here we go print("+++ Execute plot function +++") if dask: data = da.data frame_axis = da.get_axis_num(framedim) drop_axis = [da.get_axis_num(a) for a in da.dims if not a == framedim] chunks = list(data.shape) chunks[frame_axis] = 1 data = data.rechunk(chunks) if progbar: pbar = ProgressBar() pbar.register() data.map_blocks( FramePrint, chunks=[1], drop_axis=drop_axis, dtype=np.float64, dask=dask, frame_axis=frame_axis, odir=odir, cmap=cmap, clim=clim, framewidth=framewidth, frameheight=frameheight, bgcolor=bgcolor, plot_style=plot_style, lons=lons, lats=lats, time=time, norm=norm, dpi=dpi, ).compute(get=get) if progbar: pbar.unregister() # The .compute(get=get) line is some dask 'magic': it parallelizes the # print function with processes and not threads,which is a lot faster # for custom functions apparently! else: # do it with a simple for loop...can this really be quicker? print("This is slow! Do it in dask!") for ii in range(0, len(da.time)): start_time = time.time() da_slice = da[{framedim: ii}] # fig,ax,h = FramePrint(da_slice, FramePrint( da_slice, frame=ii, odir=odir, cmap=cmap, clim=clim, framewidth=framewidth, frameheight=dpi, bgcolor=bgcolor, plot_style=plot_style, lons=lons, lats=lats, norm=norm, dpi=dpi, ) if ii % 100 == 0: remaining_time = (len(da.time) - ii) * (time.time() - start_time) / 60 print("FRAME---%04d---" % ii) print("Estimated time left : %d minutes" % remaining_time) query = ('ffmpeg -y -i "frame_%05d.png" -c:v libx264 -preset veryslow \ -crf 6 -pix_fmt yuv420p \ -framerate 10 \ "' + moviename + '.mp4"') with cd(odir): if ffmpeg: print("+++ Convert frames to video +++") excode = os.system(query) if excode == 0 and delete: os.system("rm *.png")
X, Y = self.make_dataset(table=self.t, var='SB', additional=['L_MAX', 'LAYER', 'SOL_Z']) X, Y = self.duplicate_dataset(X, Y) self.SB = continuous.model('SB', X, Y, x, logger=logger, load_save=True) del X, Y X, Y = self.make_dataset(table=self.t, var='CS', additional=['L_MAX', 'LAYER', 'SOL_Z', 'SOL_SAND']) X, Y = self.duplicate_dataset(X, Y) x = self.modify_dataset(x, series=self.SOL_Z.Y_mod, name='SOL_SAND') self.CS = continuous.model('CS', X, Y, x, logger=logger, load_save=True) del X, Y X, Y = self.make_dataset(table=self.t, var='FS', additional=['L_MAX', 'LAYER', 'SOL_Z', 'SOL_SAND']) X, Y = self.duplicate_dataset(X, Y) self.FS = continuous.model('FS', X, Y, x, logger=logger, load_save=True) del X, Y, x self.write_results() if __name__ == '__main__': from dask.diagnostics import ProgressBar from dask import config from multiprocessing import freeze_support freeze_support() pbar = ProgressBar() pbar.register() config.set(scheduler='processes') main(r"C:\Users\putzr\Documents\GitHub\sleepy\model\training.txt", r"C:\Users\putzr\Documents\GitHub\sleepy\model\modelling.txt").run() pbar.unregister()
def get(*args, **kwargs): pbar = ProgressBar() pbar.register() out = client.get(*args, **kwargs) pbar.unregister() return out