def check_err_and_placeholders(result, value):
err, placeholders = result
try:
wait(value)
except TaskError:
if err is None:
err = value
else:
err = first(err, value)
except SuspendTask:
placeholders = True
return err, placeholders
def _workflow_wrapper(self, factory, input_data, *extra_args):
wf_kwargs = {}
for dep_name, proxy in self.proxy_factory_registry.items():
wf_kwargs[dep_name] = proxy(*extra_args)
func = factory(**wf_kwargs)
try:
args, kwargs = self.deserialize_input(input_data)
except Exception:
logger.exception('Cannot deserialize the input:')
raise ValueError('Cannot deserialize the input: %r' % (input_data,))
result = func(*args, **kwargs)
# Can't use directly isinstance(result, restart_type) because if the
# result is a single result proxy it will be evaluated. This also
# fixes another issue, on python2 isinstance() swallows any
# exception while python3 it doesn't.
if not is_result_proxy(result) and isinstance(result, restart_type):
try:
traversed_input, (error, placeholders) = traverse_data(
[result.args, result.kwargs])
except Exception:
logger.exception('Cannot traverse the restart arguments:')
raise ValueError(
'Cannot traverse the restart arguments: %r, %r' %
result.args, result.kwargs)
wait(error) # raise if not None
if placeholders:
raise SuspendTask
r_args, r_kwargs = traversed_input
try:
serialized_input = self.serialize_restart_input(*r_args, **r_kwargs)
except Exception:
logger.exception('Cannot serialize the restart arguments:')
raise ValueError(
'Cannot serialize the restart arguments: %r, %r' %
result.args, result.kwargs)
raise Restart(serialized_input)
try:
traversed_result, (error, placeholders) = traverse_data(result)
except Exception:
logger.exception('Cannot traverse the result:')
raise ValueError('Cannot traverse the result: %r' % result)
wait(error)
if placeholders:
raise SuspendTask
try:
return self.serialize_result(traversed_result)
except Exception:
logger.exception('Cannot serialize the result:')
raise ValueError('Cannot serialize the result: %r' % (result,))
def collect_err_and_results(result, value):
err, results = result
if not is_result_proxy(value):
return result
try:
wait(value)
except TaskError:
if err is None:
err = value
else:
err = first(err, value)
except SuspendTask:
pass
else:
if results is None:
results = []
results.append(value)
return err, results
def traverse_data(value, f=check_err_and_placeholders, initial=(None, False), seen=frozenset(), make_list=True):
if is_result_proxy(value):
try:
wait(value)
except TaskError:
return value, f(initial, value)
except SuspendTask:
return value, f(initial, value)
return value.__wrapped__, f(initial, value)
if isinstance(value, (bytes, uni)):
return value, f(initial, value)
res = initial
if isinstance(value, collections.Iterable):
if id(value) in seen:
raise ValueError('Recursive structure.')
seen = seen | frozenset([id(value)])
if isinstance(value, collections.Mapping):
d = {}
for k, v in value.items():
k_, res = traverse_data(k, f, res, seen, make_list=False)
v_, res = traverse_data(v, f, res, seen, make_list=make_list)
d[k_] = v_
return d, res
if (
isinstance(value, collections.Iterable)
and isinstance(value, collections.Sized)
):
l = []
for x in value:
x_, res = traverse_data(x, f, res, seen, make_list=make_list)
l.append(x_)
if make_list:
return l, res
return tuple(l), res
if isinstance(value, collections.Iterable):
raise ValueError('Unsized iterables not allowed.')
return value, f(initial, value)
def scan_args(args, kwargs):
"""Return a tuple of errs, placeholders.
Where errs is a list of proxy_results containing erros; and placeholders is
a boolean flag indictating whether there is at least one placeholder.
"""
errs = []
placeholders = False
for result in args:
try:
wait(result)
except SuspendTask:
placeholders = True
except Exception:
errs.append(result)
for key, result in kwargs.items():
try:
wait(result)
except SuspendTask:
placeholders = True
except Exception:
errs.append(result)
return errs, placeholders
