def async_event_sender(live_settings):
mp = get_mp_context(get_start_method())
events_queue = mp.Queue()
process = mp.Process(target=async_send, args=(events_queue, live_settings))
process.start()
return lambda event: events_queue.put(event)
def get_context(method="loky"):
if method == "fork":
warnings.warn(
"`fork` start method should not be used with `loky` "
"as it does not respect POSIX. Try using `spawn` or "
"`loky` instead.", UserWarning)
return get_mp_context(method)
def __call__(self):
fps = (os.path.join(self.fp_dir, f'{fn}.npy')
for fn in self.arrays.keys())
with ProcessPoolExecutor(
mp_context=get_mp_context('spawn')) as executor:
post = tuple(executor.map(self.func, fps, chunksize=CHUNK_SIZE))
assert len(post) == COUNT_ARRAY
def __call__(self):
fp = self.fp_dir
func = partial(self.func, fp)
with ProcessPoolExecutor(mp_context=get_mp_context('spawn')) as executor:
post = tuple(executor.map(func, self.selections, chunksize=CHUNK_SIZE))
assert self.fixture.shape[0] == post[0]
assert len(post) == len(self.selections)
def agent_function(f: Callable,
name: Optional[str] = None,
with_state: bool = False) -> Callable:
mp = get_mp_context("fork")
def wrapped(*args, **kwargs):
try:
f_in_action = inside_action(f, name=name, with_state=with_state)
return mp.Process(target=f_in_action, args=args, kwargs=kwargs)
except Exception as e:
logging.exception(f"Error during the execution of {f}: <{e}>")
return wrapped
def run(statement, settings, timeout=None, **kwargs):
mp = get_mp_context(get_start_method())
with start_action(action_type="query.run", statement=statement):
live_settings = settings["live"]
channels = start(statement, settings, timeout=timeout, **kwargs)
logging.debug(f"Results channel is {channels}")
live_url = live_settings["url"]
results_url = f"{live_url}/cometd"
events_queue = mp.Queue()
process = mp.Process(target=watch, args=(results_url, channels, events_queue, settings))
process.start()
return process, events_queue
def __call__(self):
with ProcessPoolExecutor(
mp_context=get_mp_context('spawn')) as executor:
post = tuple(
executor.map(work, self.arrays.values(), chunksize=CHUNK_SIZE))
assert len(post) == COUNT_ARRAY
def get_context(method="loky"):
if method == "fork":
warnings.warn("`fork` start method should not be used with `loky` "
"as it does not respect POSIX. Try using `spawn` or "
"`loky` instead.", UserWarning)
return get_mp_context(method)
def __call__(self):
with ProcessPoolExecutor(mp_context=get_mp_context('spawn')) as executor:
args = ((self.fixture, sel) for sel in self.selections)
post = tuple(executor.map(self.func, args, chunksize=CHUNK_SIZE))
assert self.fixture.shape[0] == post[0]
assert len(post) == len(self.selections)