def closure(rolling, func, *args, **kwargs):
series = rolling.obj
window = rolling.window
chunks = _chunk(len(series), nb_workers, window)
object_id = plasma_client.put(series)
attribute2value = {attribute: getattr(rolling, attribute)
for attribute in rolling._attributes}
with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
futures = [
executor.submit(_SeriesRolling.worker,
plasma_store_name, num, object_id,
attribute2value,
chunk, func, progress_bar,
*args, **kwargs)
for num, chunk in enumerate(chunks)
]
result = _pd.concat([
plasma_client.get(future.result())
for future in futures
], copy=False)
return result
def closure(df, func, *args, **kwargs):
axis = kwargs.get("axis", 0)
if axis == 'index':
axis = 0
elif axis == 'columns':
axis = 1
opposite_axis = 1 - axis
chunks = _chunk(df.shape[opposite_axis], nb_workers)
object_id = plasma_client.put(df)
with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
futures = [
executor.submit(_DataFrame.worker_apply, plasma_store_name,
object_id, chunk, func, progress_bar,
*args, **kwargs)
for index, chunk in enumerate(chunks)
]
result = _pd.concat(
[plasma_client.get(future.result()) for future in futures],
copy=False)
return result
class _Dialog(_EventDispatcher):
"""Dialog base class
This base class sets up a ProcessPoolExecutor with a single
background Process. This allows the Dialog to display in
the background without blocking or interfering with the main
application Process. This also limits to a single open Dialog
at a time.
"""
executor = _ProcessPoolExecutor(max_workers=1)
_dialog = None
@staticmethod
def _open_dialog(dialog):
import tkinter as tk
root = tk.Tk()
root.withdraw()
return dialog.show()
def open(self):
future = self.executor.submit(self._open_dialog, self._dialog)
future.add_done_callback(self._dispatch_event)
def _dispatch_event(self, future):
raise NotImplementedError
def fit_z_parallel(locs,
info,
calibration,
magnification_factor,
filter=2,
asynch=False):
n_workers = max(1, int(0.75 * _multiprocessing.cpu_count()))
n_locs = len(locs)
n_tasks = 100 * n_workers
spots_per_task = [
int(n_locs / n_tasks + 1) if _ < n_locs % n_tasks else int(n_locs /
n_tasks)
for _ in range(n_tasks)
]
start_indices = _np.cumsum([0] + spots_per_task[:-1])
fs = []
executor = _ProcessPoolExecutor(n_workers)
for i, n_locs_task in zip(start_indices, spots_per_task):
fs.append(
executor.submit(
fit_z,
locs[i:i + n_locs_task],
info,
calibration,
magnification_factor,
filter=0,
))
if asynch:
return fs
with _tqdm(total=n_tasks, unit="task") as progress_bar:
for f in _futures.as_completed(fs):
progress_bar.update()
return locs_from_futures(fs, filter=filter)
def closure(series, func, *args, **kwargs):
chunks = _chunk(series.size, nb_workers)
object_id = plasma_client.put(series)
with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
futures = [
executor.submit(_Series.worker_apply, plasma_store_name,
object_id, chunk, func, progress_bar,
*args, **kwargs) for chunk in chunks
]
result = _pd.concat(
[plasma_client.get(future.result()) for future in futures],
copy=False)
return result
def closure(data, func):
chunks = _chunk(data.size, nb_workers)
object_id = plasma_client.put(data)
with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
futures = [
executor.submit(_Series.worker, plasma_store_name,
object_id, _chunk, func)
for _chunk in chunks
]
result = _pd.concat(
[plasma_client.get(future.result()) for future in futures],
copy=False)
return result
def closure(data, func, **kwargs):
keys = data.groups.keys()
with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
futures = [
executor.submit(_DataFrameGroupBy.worker,
plasma_store_name,
plasma_client.put(data.get_group(key)),
func) for key in keys
]
result = _pd.DataFrame(
[plasma_client.get(future.result()) for future in futures],
index=_pd.Series(list(data.grouper), name=data.keys))
return result
def closure(df, func):
chunks = _chunk(df.shape[0], nb_workers)
object_id = plasma_client.put(df)
with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
futures = [
executor.submit(_DataFrame.worker_applymap,
plasma_store_name, object_id, chunk, func,
progress_bar)
for index, chunk in enumerate(chunks)
]
result = _pd.concat(
[plasma_client.get(future.result()) for future in futures],
copy=False)
return result
def closure(df_grouped, func, *args, **kwargs):
groups = list(df_grouped.groups.items())
chunks = _chunk(len(groups), nb_workers)
object_id = plasma_client.put(df_grouped.obj)
groups_id = plasma_client.put(groups)
with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
futures = [
executor.submit(_DataFrameGroupBy.worker,
plasma_store_name, object_id, groups_id,
chunk, func, *args, **kwargs)
for chunk in chunks
]
result = _pd.DataFrame(list(
itertools.chain.from_iterable([
plasma_client.get(future.result()) for future in futures
])),
index=_pd.Series(
list(df_grouped.grouper),
name=df_grouped.keys)).squeeze()
return result
def closure(rolling_groupby, func, *args, **kwargs):
groups = list(rolling_groupby._groupby.groups.items())
chunks = _chunk(len(groups), nb_workers)
object_id = plasma_client.put(rolling_groupby.obj)
groups_id = plasma_client.put(groups)
attribute2value = {attribute: getattr(rolling_groupby, attribute)
for attribute in rolling_groupby._attributes}
with _ProcessPoolExecutor(max_workers=nb_workers) as executor:
futures = [
executor.submit(_RollingGroupby.worker, plasma_store_name,
object_id, groups_id, attribute2value,
chunk, func, *args, **kwargs)
for chunk in chunks
]
result = _pd.concat([
plasma_client.get(future.result())
for future in futures
], copy=False)
return result
square_d_zcalib[i] = result.fun
z *= magnification_factor
locs = _lib.append_to_rec(locs, z, 'z')
locs = _lib.append_to_rec(locs, _np.sqrt(square_d_zcalib), 'd_zcalib')
locs = _lib.ensure_sanity(locs, info)
return filter_z_fits(locs, filter)
def fit_z_parallel(locs, info, calibration, magnification_factor, filter=2, async=False):
n_workers = max(1, int(0.75 * _multiprocessing.cpu_count()))
n_locs = len(locs)
n_tasks = 100 * n_workers
spots_per_task = [int(n_locs / n_tasks + 1) if _ < n_locs % n_tasks else int(n_locs / n_tasks) for _ in range(n_tasks)]
start_indices = _np.cumsum([0] + spots_per_task[:-1])
fs = []
executor = _ProcessPoolExecutor(n_workers)
for i, n_locs_task in zip(start_indices, spots_per_task):
fs.append(executor.submit(fit_z, locs[i:i+n_locs_task], info, calibration, magnification_factor, filter=0))
if async:
return fs
with _tqdm(total=n_tasks, unit='task') as progress_bar:
for f in _futures.as_completed(fs):
progress_bar.update()
return locs_from_futures(fs, filter=filter)
def locs_from_futures(futures, filter=2):
locs = [_.result() for _ in futures]
locs = _np.hstack(locs).view(_np.recarray)
return filter_z_fits(locs, filter)
