def test_required_positional_parameters_not_missing():
positionals = ["bar", "baz"]
fn_params = get_function_params(decorated_function_two_positionals_one_kwarg())
assert {fn_param.name for fn_param in fn_params} == {"bar", "qux", "baz"}
assert not validate_expected_params(fn_params, positionals)
fn_params = get_function_params(decorated_function_one_positional())
assert validate_expected_params(fn_params, positionals) == "baz"
def build_asset_ins(
fn: Callable,
asset_namespace: Optional[Sequence[str]],
asset_ins: Mapping[str, AssetIn],
non_argument_deps: Optional[Set[AssetKey]],
) -> Dict[str, In]:
non_argument_deps = check.opt_set_param(non_argument_deps,
"non_argument_deps", AssetKey)
params = get_function_params(fn)
is_context_provided = len(params) > 0 and params[
0].name in get_valid_name_permutations("context")
input_param_names = [
input_param.name
for input_param in (params[1:] if is_context_provided else params)
]
all_input_names = set(input_param_names) | asset_ins.keys()
for in_key in asset_ins.keys():
if in_key not in input_param_names:
raise DagsterInvalidDefinitionError(
f"Key '{in_key}' in provided ins dict does not correspond to any of the names "
"of the arguments to the decorated function")
ins: Dict[str, In] = {}
for input_name in all_input_names:
asset_key = None
if input_name in asset_ins:
asset_key = asset_ins[input_name].asset_key
metadata = asset_ins[input_name].metadata or {}
namespace = asset_ins[input_name].namespace
dagster_type = None
else:
metadata = {}
namespace = None
dagster_type = None
asset_key = asset_key or AssetKey(
list(
filter(None,
[*(namespace or asset_namespace or []), input_name])))
ins[input_name] = In(
metadata=metadata,
root_manager_key="root_manager",
asset_key=asset_key,
dagster_type=dagster_type,
)
for asset_key in non_argument_deps:
stringified_asset_key = "_".join(asset_key.path)
if stringified_asset_key:
ins[stringified_asset_key] = In(dagster_type=Nothing,
asset_key=asset_key)
return ins
def result(init_context):
type_storages = {
type_: storage_def.resource_fn(init_context)
if is_context_provided(get_function_params(
storage_def.resource_fn)) else storage_def.resource_fn()
for type_, storage_def in type_storage_defs.items()
}
for storage in type_storages.values():
check.invariant(
isinstance(storage, AssetStorage),
"Storages within multi-type asset storages must be AssetStorages",
)
return MultiTypeAssetStorage(type_storages)
def asset(fn):
asset_name = fn.__name__
ins: Dict[str, In] = {}
for input_param in get_function_params(fn):
input_param_name = input_param.name
asset_key = AssetKey(input_param_name)
ins[input_param_name] = In(asset_key=asset_key)
out = Out(asset_key=AssetKey(asset_name))
return _Op(
name=asset_name,
ins=ins,
out=out,
)(fn)
def wrapper(func):
params = get_function_params(func)
missing_positional = validate_expected_params(params, EXPECTED_POSITIONALS)
if missing_positional:
raise DagsterInvalidDefinitionError(
"@dagster_type_loader '{solid_name}' decorated function does not have required positional "
"parameter '{missing_param}'. @dagster_type_loader decorated functions should only have keyword arguments "
"that match input names and a first positional parameter named 'context'.".format(
solid_name=func.__name__, missing_param=missing_positional
)
)
return _create_type_loader_for_decorator(
config_type, func, required_resource_keys, loader_version, external_version_fn
)
def single_resource_event_generator(context, resource_name, resource_def):
try:
msg_fn = lambda: "Error executing resource_fn on ResourceDefinition {name}".format(
name=resource_name)
with user_code_error_boundary(DagsterResourceFunctionError,
msg_fn,
log_manager=context.log):
try:
with time_execution_scope() as timer_result:
resource_or_gen = (resource_def.resource_fn(context)
if is_context_provided(
get_function_params(
resource_def.resource_fn)) else
resource_def.resource_fn())
# Flag for whether resource is generator. This is used to ensure that teardown
# occurs when resources are initialized out of execution.
is_gen = inspect.isgenerator(
resource_or_gen) or isinstance(resource_or_gen,
ContextDecorator)
resource_iter = _wrapped_resource_iterator(resource_or_gen)
resource = next(resource_iter)
resource = InitializedResource(
resource, format_duration(timer_result.millis), is_gen)
except StopIteration:
check.failed(
"Resource generator {name} must yield one item.".format(
name=resource_name))
yield resource
except DagsterUserCodeExecutionError as dagster_user_error:
raise dagster_user_error
with user_code_error_boundary(DagsterResourceFunctionError,
msg_fn,
log_manager=context.log):
try:
next(resource_iter)
except StopIteration:
pass
else:
check.failed(
"Resource generator {name} yielded more than one item.".format(
name=resource_name))
def test_one_required_positional_param():
positionals = ["bar"]
fn_params = get_function_params(decorated_function_one_positional())
assert {fn_param.name for fn_param in fn_params} == {"bar"}
assert not validate_expected_params(fn_params, positionals)