def closure(rolling, func, *args, **kwargs): series = rolling.obj window = rolling.window chunks = _chunk(len(series), nb_workers, window) object_id = plasma_client.put(series) attribute2value = {attribute: getattr(rolling, attribute) for attribute in rolling._attributes} with _ProcessPoolExecutor(max_workers=nb_workers) as executor: futures = [ executor.submit(_SeriesRolling.worker, plasma_store_name, num, object_id, attribute2value, chunk, func, progress_bar, *args, **kwargs) for num, chunk in enumerate(chunks) ] result = _pd.concat([ plasma_client.get(future.result()) for future in futures ], copy=False) return result
def closure(df, func, *args, **kwargs): axis = kwargs.get("axis", 0) if axis == 'index': axis = 0 elif axis == 'columns': axis = 1 opposite_axis = 1 - axis chunks = _chunk(df.shape[opposite_axis], nb_workers) object_id = plasma_client.put(df) with _ProcessPoolExecutor(max_workers=nb_workers) as executor: futures = [ executor.submit(_DataFrame.worker_apply, plasma_store_name, object_id, chunk, func, progress_bar, *args, **kwargs) for index, chunk in enumerate(chunks) ] result = _pd.concat( [plasma_client.get(future.result()) for future in futures], copy=False) return result
class _Dialog(_EventDispatcher): """Dialog base class This base class sets up a ProcessPoolExecutor with a single background Process. This allows the Dialog to display in the background without blocking or interfering with the main application Process. This also limits to a single open Dialog at a time. """ executor = _ProcessPoolExecutor(max_workers=1) _dialog = None @staticmethod def _open_dialog(dialog): import tkinter as tk root = tk.Tk() root.withdraw() return dialog.show() def open(self): future = self.executor.submit(self._open_dialog, self._dialog) future.add_done_callback(self._dispatch_event) def _dispatch_event(self, future): raise NotImplementedError
def fit_z_parallel(locs, info, calibration, magnification_factor, filter=2, asynch=False): n_workers = max(1, int(0.75 * _multiprocessing.cpu_count())) n_locs = len(locs) n_tasks = 100 * n_workers spots_per_task = [ int(n_locs / n_tasks + 1) if _ < n_locs % n_tasks else int(n_locs / n_tasks) for _ in range(n_tasks) ] start_indices = _np.cumsum([0] + spots_per_task[:-1]) fs = [] executor = _ProcessPoolExecutor(n_workers) for i, n_locs_task in zip(start_indices, spots_per_task): fs.append( executor.submit( fit_z, locs[i:i + n_locs_task], info, calibration, magnification_factor, filter=0, )) if asynch: return fs with _tqdm(total=n_tasks, unit="task") as progress_bar: for f in _futures.as_completed(fs): progress_bar.update() return locs_from_futures(fs, filter=filter)
def closure(series, func, *args, **kwargs): chunks = _chunk(series.size, nb_workers) object_id = plasma_client.put(series) with _ProcessPoolExecutor(max_workers=nb_workers) as executor: futures = [ executor.submit(_Series.worker_apply, plasma_store_name, object_id, chunk, func, progress_bar, *args, **kwargs) for chunk in chunks ] result = _pd.concat( [plasma_client.get(future.result()) for future in futures], copy=False) return result
def closure(data, func): chunks = _chunk(data.size, nb_workers) object_id = plasma_client.put(data) with _ProcessPoolExecutor(max_workers=nb_workers) as executor: futures = [ executor.submit(_Series.worker, plasma_store_name, object_id, _chunk, func) for _chunk in chunks ] result = _pd.concat( [plasma_client.get(future.result()) for future in futures], copy=False) return result
def closure(data, func, **kwargs): keys = data.groups.keys() with _ProcessPoolExecutor(max_workers=nb_workers) as executor: futures = [ executor.submit(_DataFrameGroupBy.worker, plasma_store_name, plasma_client.put(data.get_group(key)), func) for key in keys ] result = _pd.DataFrame( [plasma_client.get(future.result()) for future in futures], index=_pd.Series(list(data.grouper), name=data.keys)) return result
def closure(df, func): chunks = _chunk(df.shape[0], nb_workers) object_id = plasma_client.put(df) with _ProcessPoolExecutor(max_workers=nb_workers) as executor: futures = [ executor.submit(_DataFrame.worker_applymap, plasma_store_name, object_id, chunk, func, progress_bar) for index, chunk in enumerate(chunks) ] result = _pd.concat( [plasma_client.get(future.result()) for future in futures], copy=False) return result
def closure(df_grouped, func, *args, **kwargs): groups = list(df_grouped.groups.items()) chunks = _chunk(len(groups), nb_workers) object_id = plasma_client.put(df_grouped.obj) groups_id = plasma_client.put(groups) with _ProcessPoolExecutor(max_workers=nb_workers) as executor: futures = [ executor.submit(_DataFrameGroupBy.worker, plasma_store_name, object_id, groups_id, chunk, func, *args, **kwargs) for chunk in chunks ] result = _pd.DataFrame(list( itertools.chain.from_iterable([ plasma_client.get(future.result()) for future in futures ])), index=_pd.Series( list(df_grouped.grouper), name=df_grouped.keys)).squeeze() return result
def closure(rolling_groupby, func, *args, **kwargs): groups = list(rolling_groupby._groupby.groups.items()) chunks = _chunk(len(groups), nb_workers) object_id = plasma_client.put(rolling_groupby.obj) groups_id = plasma_client.put(groups) attribute2value = {attribute: getattr(rolling_groupby, attribute) for attribute in rolling_groupby._attributes} with _ProcessPoolExecutor(max_workers=nb_workers) as executor: futures = [ executor.submit(_RollingGroupby.worker, plasma_store_name, object_id, groups_id, attribute2value, chunk, func, *args, **kwargs) for chunk in chunks ] result = _pd.concat([ plasma_client.get(future.result()) for future in futures ], copy=False) return result
square_d_zcalib[i] = result.fun z *= magnification_factor locs = _lib.append_to_rec(locs, z, 'z') locs = _lib.append_to_rec(locs, _np.sqrt(square_d_zcalib), 'd_zcalib') locs = _lib.ensure_sanity(locs, info) return filter_z_fits(locs, filter) def fit_z_parallel(locs, info, calibration, magnification_factor, filter=2, async=False): n_workers = max(1, int(0.75 * _multiprocessing.cpu_count())) n_locs = len(locs) n_tasks = 100 * n_workers spots_per_task = [int(n_locs / n_tasks + 1) if _ < n_locs % n_tasks else int(n_locs / n_tasks) for _ in range(n_tasks)] start_indices = _np.cumsum([0] + spots_per_task[:-1]) fs = [] executor = _ProcessPoolExecutor(n_workers) for i, n_locs_task in zip(start_indices, spots_per_task): fs.append(executor.submit(fit_z, locs[i:i+n_locs_task], info, calibration, magnification_factor, filter=0)) if async: return fs with _tqdm(total=n_tasks, unit='task') as progress_bar: for f in _futures.as_completed(fs): progress_bar.update() return locs_from_futures(fs, filter=filter) def locs_from_futures(futures, filter=2): locs = [_.result() for _ in futures] locs = _np.hstack(locs).view(_np.recarray) return filter_z_fits(locs, filter)