def test_get_args():
def foo(one, two=2, three=None): # pylint: disable=unused-argument
pass
assert len(seven.get_args(foo)) == 3
assert 'one' in seven.get_args(foo)
assert 'two' in seven.get_args(foo)
assert 'three' in seven.get_args(foo)
def _unpack_value(val, enum_map, tuple_map):
if isinstance(val, list):
return [_unpack_value(i, enum_map, tuple_map) for i in val]
if isinstance(val, dict) and val.get('__class__'):
klass_name = val.pop('__class__')
klass = tuple_map[klass_name]
val = {
key: _unpack_value(value, enum_map, tuple_map)
for key, value in val.items()
}
# Naively implements backwards compatibility by filtering arguments that aren't present in
# the constructor. If a property is present in the serialized object, but doesn't exist in
# the version of the class loaded into memory, that property will be completely ignored.
args_for_class = seven.get_args(klass)
filtered_val = ({k: v
for k, v in val.items()
if k in args_for_class} if args_for_class else val)
return klass(**filtered_val)
if isinstance(val, dict) and val.get('__enum__'):
name, member = val['__enum__'].split('.')
return getattr(enum_map[name], member)
if isinstance(val, dict):
return {
key: _unpack_value(value, enum_map, tuple_map)
for key, value in val.items()
}
return val
def _validate_type_check_fn(fn: t.Callable, name: t.Optional[str]) -> bool:
from dagster.seven import get_args
args = get_args(fn)
# py2 doesn't filter out self
if len(args) >= 1 and args[0] == "self":
args = args[1:]
if len(args) == 2:
possible_names = {
"_",
"context",
"_context",
"context_",
}
if args[0] not in possible_names:
DagsterInvalidDefinitionError(
'type_check function on type "{name}" must have first '
'argument named "context" (or _, _context, context_).'.format(
name=name,
)
)
return True
raise DagsterInvalidDefinitionError(
'type_check_fn argument on type "{name}" must take 2 arguments, '
"received {count}.".format(name=name, count=len(args))
)
def def_from_pointer(
pointer: CodePointer,
) -> Union["PipelineDefinition", "RepositoryDefinition", "GraphDefinition"]:
target = pointer.load_target()
from .pipeline_definition import PipelineDefinition
from .repository_definition import RepositoryDefinition
from .graph_definition import GraphDefinition
if isinstance(
target, (PipelineDefinition, RepositoryDefinition, GraphDefinition)
) or not callable(target):
return _check_is_loadable(target)
# if its a function invoke it - otherwise we are pointing to a
# artifact in module scope, likely decorator output
if seven.get_args(target):
raise DagsterInvariantViolationError(
"Error invoking function at {target} with no arguments. "
"Reconstructable target must be callable with no arguments".format(
target=pointer.describe()
)
)
return _check_is_loadable(target())
def _validate_type_check_fn(fn, name):
from dagster.seven import get_args
args = get_args(fn)
# py2 doesn't filter out self
if len(args) >= 1 and args[0] == 'self':
args = args[1:]
if len(args) == 2:
possible_names = {
'_',
'context',
'_context',
'context_',
}
if args[0] not in possible_names:
DagsterInvalidDefinitionError(
'type_check function on type "{name}" must have first '
'argument named "context" (or _, _context, context_).'.format(
name=name, ))
return True
raise DagsterInvalidDefinitionError(
'type_check_fn argument on type "{name}" must take 2 arguments, '
'received {count}.'.format(name=name, count=len(args)))
def _unpack_value(val, enum_map, tuple_map):
if isinstance(val, list):
return [_unpack_value(i, enum_map, tuple_map) for i in val]
if isinstance(val, dict) and val.get('__class__'):
klass_name = val.pop('__class__')
klass = tuple_map[klass_name]
unpacked_val = {
key: _unpack_value(value, enum_map, tuple_map)
for key, value in val.items()
}
# Naively implements backwards compatibility by filtering arguments that aren't present in
# the constructor. If a property is present in the serialized object, but doesn't exist in
# the version of the class loaded into memory, that property will be completely ignored.
# The call to seven.get_args turns out to be pretty expensive -- we should probably turn
# to, e.g., manually managing the deprecated keys on the serdes constructor.
args_for_class = seven.get_args(klass)
filtered_val = {
k: v
for k, v in unpacked_val.items() if k in args_for_class
}
return klass(**filtered_val)
if isinstance(val, dict) and val.get('__enum__'):
name, member = val['__enum__'].split('.')
return getattr(enum_map[name], member)
if isinstance(val, dict):
return {
key: _unpack_value(value, enum_map, tuple_map)
for key, value in val.items()
}
return val
def value_from_storage_dict(cls, storage_dict: Dict[str, Any], klass: Type) -> NamedTuple:
# Naively implements backwards compatibility by filtering arguments that aren't present in
# the constructor. If a property is present in the serialized object, but doesn't exist in
# the version of the class loaded into memory, that property will be completely ignored.
# The call to seven.get_args turns out to be pretty expensive -- we should probably turn
# to, e.g., manually managing the deprecated keys on the serdes constructor.
args_for_class = seven.get_args(klass)
filtered_val = {k: v for k, v in storage_dict.items() if k in args_for_class}
return klass(**filtered_val)
def _unpack_value(val, whitelist_map):
if isinstance(val, list):
return [_unpack_value(i, whitelist_map) for i in val]
if isinstance(val, dict) and val.get("__class__"):
klass_name = val.pop("__class__")
check.invariant(
klass_name in whitelist_map["types"]["tuple"],
(f'Attempted to deserialize class "{klass_name}", which is not in '
"the serdes whitelist."),
)
klass = whitelist_map["types"]["tuple"][klass_name]
if klass is None:
return None
unpacked_val = {
key: _unpack_value(value, whitelist_map)
for key, value in val.items()
}
if klass_name in whitelist_map["persistence"]:
return klass.from_storage_dict(unpacked_val)
# Naively implements backwards compatibility by filtering arguments that aren't present in
# the constructor. If a property is present in the serialized object, but doesn't exist in
# the version of the class loaded into memory, that property will be completely ignored.
# The call to seven.get_args turns out to be pretty expensive -- we should probably turn
# to, e.g., manually managing the deprecated keys on the serdes constructor.
args_for_class = seven.get_args(klass)
filtered_val = {
k: v
for k, v in unpacked_val.items() if k in args_for_class
}
return klass(**filtered_val)
if isinstance(val, dict) and val.get("__enum__"):
name, member = val["__enum__"].split(".")
return getattr(whitelist_map["types"]["enum"][name], member)
if isinstance(val, dict) and val.get("__set__") is not None:
return set(
[_unpack_value(item, whitelist_map) for item in val["__set__"]])
if isinstance(val, dict) and val.get("__frozenset__") is not None:
return frozenset([
_unpack_value(item, whitelist_map) for item in val["__frozenset__"]
])
if isinstance(val, dict):
return {
key: _unpack_value(value, whitelist_map)
for key, value in val.items()
}
return val
def def_from_pointer(pointer):
target = pointer.load_target()
if not callable(target):
return _check_is_loadable(target)
# if its a function invoke it - otherwise we are pointing to a
# artifact in module scope, likely decorator output
if seven.get_args(target):
raise DagsterInvariantViolationError(
"Error invoking function at {target} with no arguments. "
"Reconstructable target must be callable with no arguments".format(
target=pointer.describe()))
return _check_is_loadable(target())
