def _callable_executor_(self, scope, view_args, output_types, provenance):
outputs = self._callable(scope.ctx, **view_args)
outputs = tuplize(outputs)
for output in outputs:
if not isinstance(output, qiime2.sdk.Result):
raise TypeError("Pipelines must return Result objects, not %r"
% output)
# This condition *is* tested by the caller of _callable_executor_, but
# the kinds of errors a plugin developer see will make more sense if
# this check happens before the subtype check. Otherwise forgetting an
# output would more likely error as a wrong type, which while correct,
# isn't root of the problem.
if len(outputs) != len(output_types):
raise TypeError(
"Number of outputs must match number of output "
"semantic types: %d != %d"
% (len(outputs), len(output_types)))
results = []
for output, (name, spec) in zip(outputs, output_types.items()):
if not (output.type <= spec.qiime_type):
raise TypeError(
"Expected output type %r, received %r" %
(spec.qiime_type, output.type))
prov = provenance.fork(name, output)
scope.add_reference(prov)
aliased_result = output._alias(prov)
scope.add_parent_reference(aliased_result)
results.append(aliased_result)
return tuple(results)
def _callable_executor_(self, scope, view_args, output_types, provenance):
outputs = self._callable(scope.ctx, **view_args)
outputs = tuplize(outputs)
for output in outputs:
if not isinstance(output, qiime2.sdk.Result):
raise TypeError(
"Pipelines must return Result objects, not %r" % output)
# This condition *is* tested by the caller of _callable_executor_, but
# the kinds of errors a plugin developer see will make more sense if
# this check happens before the subtype check. Otherwise forgetting an
# output would more likely error as a wrong type, which while correct,
# isn't root of the problem.
if len(outputs) != len(output_types):
raise TypeError("Number of outputs must match number of output "
"semantic types: %d != %d" %
(len(outputs), len(output_types)))
results = []
for output, (name, spec) in zip(outputs, output_types.items()):
if not (output.type <= spec.qiime_type):
raise TypeError("Expected output type %r, received %r" %
(spec.qiime_type, output.type))
prov = provenance.fork(name, output)
scope.add_reference(prov)
aliased_result = output._alias(prov)
scope.add_parent_reference(aliased_result)
results.append(aliased_result)
return tuple(results)
def _callable_executor_(self, scope, view_args, output_types, provenance):
output_views = self._callable(**view_args)
output_views = tuplize(output_views)
# TODO this won't work if the user has annotated their "view API" to
# return a `typing.Tuple` with some number of components. Python will
# return a tuple when there are multiple return values, and this length
# check will fail because the tuple as a whole should be matched up to
# a single output type instead of its components. This is an edgecase
# due to how Python handles multiple returns, and can be worked around
# by using something like `typing.List` instead.
if len(output_views) != len(output_types):
raise TypeError(
"Number of output views must match number of output "
"semantic types: %d != %d"
% (len(output_views), len(output_types)))
output_artifacts = []
for output_view, (name, spec) in zip(output_views,
output_types.items()):
if type(output_view) is not spec.view_type:
raise TypeError(
"Expected output view type %r, received %r" %
(spec.view_type.__name__, type(output_view).__name__))
prov = provenance.fork(name)
scope.add_reference(prov)
artifact = qiime2.sdk.Artifact._from_view(
spec.qiime_type, output_view, spec.view_type, prov)
scope.add_parent_reference(artifact)
output_artifacts.append(artifact)
return tuple(output_artifacts)
def _callable_executor_(self, callable, view_args, output_types,
provenance):
is_subprocess = self._is_subprocess()
output_views = callable(**view_args)
output_views = tuplize(output_views)
# TODO this won't work if the user has annotated their "view API" to
# return a `typing.Tuple` with some number of components. Python will
# return a tuple when there are multiple return values, and this length
# check will fail because the tuple as a whole should be matched up to
# a single output type instead of its components. This is an edgecase
# due to how Python handles multiple returns, and can be worked around
# by using something like `typing.List` instead.
if len(output_views) != len(output_types):
raise TypeError(
"Number of output views must match number of output "
"semantic types: %d != %d"
% (len(output_views), len(output_types)))
output_artifacts = []
for output_view, (name, spec) in zip(output_views,
output_types.items()):
if type(output_view) is not spec.view_type:
raise TypeError(
"Expected output view type %r, received %r" %
(spec.view_type.__name__, type(output_view).__name__))
artifact = qiime2.sdk.Artifact._from_view(
spec.qiime_type, output_view, spec.view_type,
provenance.fork(name))
output_artifacts.append(artifact)
if is_subprocess:
_FAILURE_PROCESS_CLEANUP.append(artifact._archiver)
return tuple(output_artifacts)
def callable_wrapper(*args, **kwargs):
provenance = archive.ActionProvenanceCapture(
self.type, self.package, self.id)
# This function's signature is rewritten below using
# `decorator.decorator`. When the signature is rewritten, args[0]
# is the function whose signature was used to rewrite this
# function's signature.
args = args[1:]
# TODO this may be able to be simplified once Signature respects
# order.
wrapper_sig = self._callable_sig_converter_(self._callable)
wrapper_sig = inspect.Signature.from_callable(wrapper_sig)
wrapper_params = wrapper_sig.parameters
user_input = {
name: value
for value, name in zip(args, wrapper_params)
}
user_input.update(kwargs)
self.signature.check_types(**user_input)
output_types = self.signature.solve_output(**user_input)
artifacts = {}
for name in self.signature.inputs:
artifact = artifacts[name] = user_input[name]
provenance.add_input(name, artifact)
parameters = {}
for name, spec in self.signature.parameters.items():
parameter = parameters[name] = user_input[name]
provenance.add_parameter(name, spec.qiime_type, parameter)
view_args = parameters.copy()
for name, spec in self.signature.inputs.items():
recorder = provenance.transformation_recorder(name)
view_args[name] = artifacts[name]._view(
spec.view_type, recorder)
outputs = self._callable_executor_(self._callable, view_args,
output_types, provenance)
# `outputs` matches a Python function's return: either a single
# value is returned, or it is a tuple of return values. Treat both
# cases uniformly.
outputs_tuple = tuplize(outputs)
for output in outputs_tuple:
output._orphan(self._pid)
if len(outputs_tuple) != len(self.signature.outputs):
raise ValueError(
"Number of callable outputs must match number of outputs "
"defined in signature: %d != %d" %
(len(outputs_tuple), len(self.signature.outputs)))
# Wrap in a Results object mapping output name to value so users
# have access to outputs by name or position.
return qiime2.sdk.Results(self.signature.outputs.keys(),
outputs_tuple)
def __getitem__(self, fields):
fields = tuplize(fields)
for field in fields:
if not isinstance(field, _AlgebraicExpBase):
raise TypeError("Field %r is not complete type expression." %
(field, ))
self.template.validate_fields_expr(self, fields)
return TypeExp(self.template, fields=fields)
def __getitem__(self, fields):
fields = tuplize(fields)
if len(fields) != len(self.field_names):
raise TypeError("%r takes %d field(s), %d provided." %
(self, len(self.field_names), len(fields)))
for args in zip(self.field_names, fields):
self._validate_field_(*args)
return self._apply_fields_(fields=fields)
def __getitem__(self, fields):
fields = tuplize(fields)
if len(fields) != len(self.field_names):
raise TypeError("%r takes %d field(s), %d provided."
% (self, len(self.field_names), len(fields)))
for args in zip(self.field_names, fields):
self._validate_field_(*args)
return self._apply_fields_(fields=fields)
def callable_wrapper(*args, **kwargs):
provenance = archive.ActionProvenanceCapture(
self.type, self.package, self.id)
# This function's signature is rewritten below using
# `decorator.decorator`. When the signature is rewritten, args[0]
# is the function whose signature was used to rewrite this
# function's signature.
args = args[1:]
# TODO this may be able to be simplified once Signature respects
# order.
wrapper_sig = self._callable_sig_converter_(self._callable)
wrapper_sig = inspect.Signature.from_callable(wrapper_sig)
wrapper_params = wrapper_sig.parameters
user_input = {name: value for value, name in
zip(args, wrapper_params)}
user_input.update(kwargs)
self.signature.check_types(**user_input)
output_types = self.signature.solve_output(**user_input)
artifacts = {}
for name in self.signature.inputs:
artifact = artifacts[name] = user_input[name]
provenance.add_input(name, artifact)
parameters = {}
for name, spec in self.signature.parameters.items():
parameter = parameters[name] = user_input[name]
provenance.add_parameter(name, spec.qiime_type, parameter)
view_args = parameters.copy()
for name, spec in self.signature.inputs.items():
recorder = provenance.transformation_recorder(name)
view_args[name] = artifacts[name]._view(spec.view_type,
recorder)
outputs = self._callable_executor_(self._callable, view_args,
output_types, provenance)
# `outputs` matches a Python function's return: either a single
# value is returned, or it is a tuple of return values. Treat both
# cases uniformly.
outputs_tuple = tuplize(outputs)
for output in outputs_tuple:
output._orphan(self._pid)
if len(outputs_tuple) != len(self.signature.outputs):
raise ValueError(
"Number of callable outputs must match number of outputs "
"defined in signature: %d != %d" %
(len(outputs_tuple), len(self.signature.outputs)))
# Wrap in a Results object mapping output name to value so users
# have access to outputs by name or position.
return qiime2.sdk.Results(self.signature.outputs.keys(),
outputs_tuple)
def parse_primitive(t, value):
expr = _norm_input(t)
result = []
allowed = None
homogeneous = True
if is_metadata_type(expr):
raise ValueError('%r may not be parsed with this util.' % (expr, ))
expr = _strip_predicates(expr)
collection_style = interrogate_collection_type(expr)
if collection_style.style in ('simple', 'monomorphic', 'composite'):
allowed = collection_style.members
if collection_style.style == 'composite':
homogeneous = False
elif collection_style.style == 'complex':
# Sort here so that we can start with any simple lists in the memberset
for subexpr in sorted(collection_style.members, key=len):
expr = collection_style.base[UnionExp(subexpr)]
try:
return parse_primitive(expr, value)
except ValueError:
pass
raise _COERCE_ERROR
elif collection_style.style is None:
value = tuplize(value)
if expr in (Int, Float, Bool, Str):
# No sense in walking over all options when we know
# what it should be
allowed = expr
else:
allowed = _COERCION_MAPPER.keys()
else:
pass
assert allowed is not None
for v in value:
result.append(_interrogate_types(allowed, v))
# Some exprs require homogeneous values, make it so
if homogeneous:
all_matching = False
for member in allowed:
if all(type(x) == _COERCION_MAPPER[member].pytype for x in result):
all_matching = True
break
if not all_matching and collection_style.style == 'monomorphic':
for subexpr in allowed:
expr = collection_style.base[subexpr]
try:
return parse_primitive(expr, value)
except ValueError:
pass
raise _COERCE_ERROR
if collection_style.view is None:
return result[0]
else:
return collection_style.view(result)
