def apply_async(self, func, callback=None):
exec_func = 'func{}{}'.format(round(time.time()),
random.randint(0, round(time.time())))
pickle_func = codecs.encode(cloudpickle.dumps(func()),
'base64').decode()
write_func = '{}/{}.csv'.format(get_file_dir(__file__), exec_func)
if self.mode == 'local':
return self._get_pool().apply_async(SafeFunction(
self._local_execute_func),
args=[
exec_func,
write_func,
pickle_func,
self._python_path,
],
callback=callback)
else:
exec_dict = {exec_func: pickle_func}
return self._get_pool().apply_async(SafeFunction(
self._remote_execute_func),
args=[
exec_func,
write_func,
exec_dict,
self._job_manager,
self._python_path,
],
callback=callback)
def test_nested_exception_dispatch():
"""Ensure TransportableException objects for nested joblib cases gets
propagated."""
with raises(JoblibException):
Parallel(n_jobs=2, pre_dispatch=16,
verbose=0)(delayed(SafeFunction(exception_raiser))(i)
for i in range(30))
def test_safe_function():
safe_division = SafeFunction(division)
if PY3_OR_LATER:
with raises(ZeroDivisionError):
safe_division(1, 0)
else:
# Under Python 2.7, exception are wrapped with a special wrapper to
# preserve runtime information of the worker environment. Python 3 does
# not need this as it preserves the traceback information by default.
with raises(TransportableException) as excinfo:
safe_division(1, 0)
assert isinstance(excinfo.value.unwrap(), ZeroDivisionError)
safe_interrupt = SafeFunction(interrupt_raiser)
with raises(WorkerInterrupt):
safe_interrupt('x')
def apply_async(self, func, callback=None):
# Note the `func` args is a batch here. (BatchedCalls type)
# See joblib.parallel.Parallel._dispatch
def run_on_worker_and_fetch_result():
# TODO: handle possible spark exception here. # pylint: disable=fixme
rdd = self._spark.sparkContext.parallelize([0], 1) \
.map(lambda _: cloudpickle.dumps(func()))
ser_res = rdd.collect()[0]
return cloudpickle.loads(ser_res)
return self._get_pool().apply_async(
SafeFunction(run_on_worker_and_fetch_result), callback=callback)
def apply_async(self, func, callback=None):
# Note the `func` args is a batch here. (BatchedCalls type)
# See joblib.parallel.Parallel._dispatch
def run_on_worker_and_fetch_result():
# TODO: handle possible spark exception here.
self._spark.sparkContext.setJobGroup(self._job_group,
"joblib spark jobs")
ser_res = self._spark.sparkContext.parallelize([0], 1) \
.map(lambda _: cloudpickle.dumps(func())) \
.first()
return cloudpickle.loads(ser_res)
return self._get_pool().apply_async(
SafeFunction(run_on_worker_and_fetch_result), callback=callback)
def test_safe_function():
safe_division = SafeFunction(division)
with raises(JoblibException):
safe_division(1, 0)
def test_safe_function():
safe_division = SafeFunction(division)
assert_raises(JoblibException, safe_division, 1, 0)