def check_err_and_placeholders(result, value): err, placeholders = result try: wait(value) except TaskError: if err is None: err = value else: err = first(err, value) except SuspendTask: placeholders = True return err, placeholders
def _workflow_wrapper(self, factory, input_data, *extra_args): wf_kwargs = {} for dep_name, proxy in self.proxy_factory_registry.items(): wf_kwargs[dep_name] = proxy(*extra_args) func = factory(**wf_kwargs) try: args, kwargs = self.deserialize_input(input_data) except Exception: logger.exception('Cannot deserialize the input:') raise ValueError('Cannot deserialize the input: %r' % (input_data,)) result = func(*args, **kwargs) # Can't use directly isinstance(result, restart_type) because if the # result is a single result proxy it will be evaluated. This also # fixes another issue, on python2 isinstance() swallows any # exception while python3 it doesn't. if not is_result_proxy(result) and isinstance(result, restart_type): try: traversed_input, (error, placeholders) = traverse_data( [result.args, result.kwargs]) except Exception: logger.exception('Cannot traverse the restart arguments:') raise ValueError( 'Cannot traverse the restart arguments: %r, %r' % result.args, result.kwargs) wait(error) # raise if not None if placeholders: raise SuspendTask r_args, r_kwargs = traversed_input try: serialized_input = self.serialize_restart_input(*r_args, **r_kwargs) except Exception: logger.exception('Cannot serialize the restart arguments:') raise ValueError( 'Cannot serialize the restart arguments: %r, %r' % result.args, result.kwargs) raise Restart(serialized_input) try: traversed_result, (error, placeholders) = traverse_data(result) except Exception: logger.exception('Cannot traverse the result:') raise ValueError('Cannot traverse the result: %r' % result) wait(error) if placeholders: raise SuspendTask try: return self.serialize_result(traversed_result) except Exception: logger.exception('Cannot serialize the result:') raise ValueError('Cannot serialize the result: %r' % (result,))
def collect_err_and_results(result, value): err, results = result if not is_result_proxy(value): return result try: wait(value) except TaskError: if err is None: err = value else: err = first(err, value) except SuspendTask: pass else: if results is None: results = [] results.append(value) return err, results
def traverse_data(value, f=check_err_and_placeholders, initial=(None, False), seen=frozenset(), make_list=True): if is_result_proxy(value): try: wait(value) except TaskError: return value, f(initial, value) except SuspendTask: return value, f(initial, value) return value.__wrapped__, f(initial, value) if isinstance(value, (bytes, uni)): return value, f(initial, value) res = initial if isinstance(value, collections.Iterable): if id(value) in seen: raise ValueError('Recursive structure.') seen = seen | frozenset([id(value)]) if isinstance(value, collections.Mapping): d = {} for k, v in value.items(): k_, res = traverse_data(k, f, res, seen, make_list=False) v_, res = traverse_data(v, f, res, seen, make_list=make_list) d[k_] = v_ return d, res if ( isinstance(value, collections.Iterable) and isinstance(value, collections.Sized) ): l = [] for x in value: x_, res = traverse_data(x, f, res, seen, make_list=make_list) l.append(x_) if make_list: return l, res return tuple(l), res if isinstance(value, collections.Iterable): raise ValueError('Unsized iterables not allowed.') return value, f(initial, value)
def scan_args(args, kwargs): """Return a tuple of errs, placeholders. Where errs is a list of proxy_results containing erros; and placeholders is a boolean flag indictating whether there is at least one placeholder. """ errs = [] placeholders = False for result in args: try: wait(result) except SuspendTask: placeholders = True except Exception: errs.append(result) for key, result in kwargs.items(): try: wait(result) except SuspendTask: placeholders = True except Exception: errs.append(result) return errs, placeholders