def test_get_args(): def foo(one, two=2, three=None): # pylint: disable=unused-argument pass assert len(seven.get_args(foo)) == 3 assert 'one' in seven.get_args(foo) assert 'two' in seven.get_args(foo) assert 'three' in seven.get_args(foo)
def _unpack_value(val, enum_map, tuple_map): if isinstance(val, list): return [_unpack_value(i, enum_map, tuple_map) for i in val] if isinstance(val, dict) and val.get('__class__'): klass_name = val.pop('__class__') klass = tuple_map[klass_name] val = { key: _unpack_value(value, enum_map, tuple_map) for key, value in val.items() } # Naively implements backwards compatibility by filtering arguments that aren't present in # the constructor. If a property is present in the serialized object, but doesn't exist in # the version of the class loaded into memory, that property will be completely ignored. args_for_class = seven.get_args(klass) filtered_val = ({k: v for k, v in val.items() if k in args_for_class} if args_for_class else val) return klass(**filtered_val) if isinstance(val, dict) and val.get('__enum__'): name, member = val['__enum__'].split('.') return getattr(enum_map[name], member) if isinstance(val, dict): return { key: _unpack_value(value, enum_map, tuple_map) for key, value in val.items() } return val
def _validate_type_check_fn(fn: t.Callable, name: t.Optional[str]) -> bool: from dagster.seven import get_args args = get_args(fn) # py2 doesn't filter out self if len(args) >= 1 and args[0] == "self": args = args[1:] if len(args) == 2: possible_names = { "_", "context", "_context", "context_", } if args[0] not in possible_names: DagsterInvalidDefinitionError( 'type_check function on type "{name}" must have first ' 'argument named "context" (or _, _context, context_).'.format( name=name, ) ) return True raise DagsterInvalidDefinitionError( 'type_check_fn argument on type "{name}" must take 2 arguments, ' "received {count}.".format(name=name, count=len(args)) )
def def_from_pointer( pointer: CodePointer, ) -> Union["PipelineDefinition", "RepositoryDefinition", "GraphDefinition"]: target = pointer.load_target() from .pipeline_definition import PipelineDefinition from .repository_definition import RepositoryDefinition from .graph_definition import GraphDefinition if isinstance( target, (PipelineDefinition, RepositoryDefinition, GraphDefinition) ) or not callable(target): return _check_is_loadable(target) # if its a function invoke it - otherwise we are pointing to a # artifact in module scope, likely decorator output if seven.get_args(target): raise DagsterInvariantViolationError( "Error invoking function at {target} with no arguments. " "Reconstructable target must be callable with no arguments".format( target=pointer.describe() ) ) return _check_is_loadable(target())
def _validate_type_check_fn(fn, name): from dagster.seven import get_args args = get_args(fn) # py2 doesn't filter out self if len(args) >= 1 and args[0] == 'self': args = args[1:] if len(args) == 2: possible_names = { '_', 'context', '_context', 'context_', } if args[0] not in possible_names: DagsterInvalidDefinitionError( 'type_check function on type "{name}" must have first ' 'argument named "context" (or _, _context, context_).'.format( name=name, )) return True raise DagsterInvalidDefinitionError( 'type_check_fn argument on type "{name}" must take 2 arguments, ' 'received {count}.'.format(name=name, count=len(args)))
def _unpack_value(val, enum_map, tuple_map): if isinstance(val, list): return [_unpack_value(i, enum_map, tuple_map) for i in val] if isinstance(val, dict) and val.get('__class__'): klass_name = val.pop('__class__') klass = tuple_map[klass_name] unpacked_val = { key: _unpack_value(value, enum_map, tuple_map) for key, value in val.items() } # Naively implements backwards compatibility by filtering arguments that aren't present in # the constructor. If a property is present in the serialized object, but doesn't exist in # the version of the class loaded into memory, that property will be completely ignored. # The call to seven.get_args turns out to be pretty expensive -- we should probably turn # to, e.g., manually managing the deprecated keys on the serdes constructor. args_for_class = seven.get_args(klass) filtered_val = { k: v for k, v in unpacked_val.items() if k in args_for_class } return klass(**filtered_val) if isinstance(val, dict) and val.get('__enum__'): name, member = val['__enum__'].split('.') return getattr(enum_map[name], member) if isinstance(val, dict): return { key: _unpack_value(value, enum_map, tuple_map) for key, value in val.items() } return val
def value_from_storage_dict(cls, storage_dict: Dict[str, Any], klass: Type) -> NamedTuple: # Naively implements backwards compatibility by filtering arguments that aren't present in # the constructor. If a property is present in the serialized object, but doesn't exist in # the version of the class loaded into memory, that property will be completely ignored. # The call to seven.get_args turns out to be pretty expensive -- we should probably turn # to, e.g., manually managing the deprecated keys on the serdes constructor. args_for_class = seven.get_args(klass) filtered_val = {k: v for k, v in storage_dict.items() if k in args_for_class} return klass(**filtered_val)
def _unpack_value(val, whitelist_map): if isinstance(val, list): return [_unpack_value(i, whitelist_map) for i in val] if isinstance(val, dict) and val.get("__class__"): klass_name = val.pop("__class__") check.invariant( klass_name in whitelist_map["types"]["tuple"], (f'Attempted to deserialize class "{klass_name}", which is not in ' "the serdes whitelist."), ) klass = whitelist_map["types"]["tuple"][klass_name] if klass is None: return None unpacked_val = { key: _unpack_value(value, whitelist_map) for key, value in val.items() } if klass_name in whitelist_map["persistence"]: return klass.from_storage_dict(unpacked_val) # Naively implements backwards compatibility by filtering arguments that aren't present in # the constructor. If a property is present in the serialized object, but doesn't exist in # the version of the class loaded into memory, that property will be completely ignored. # The call to seven.get_args turns out to be pretty expensive -- we should probably turn # to, e.g., manually managing the deprecated keys on the serdes constructor. args_for_class = seven.get_args(klass) filtered_val = { k: v for k, v in unpacked_val.items() if k in args_for_class } return klass(**filtered_val) if isinstance(val, dict) and val.get("__enum__"): name, member = val["__enum__"].split(".") return getattr(whitelist_map["types"]["enum"][name], member) if isinstance(val, dict) and val.get("__set__") is not None: return set( [_unpack_value(item, whitelist_map) for item in val["__set__"]]) if isinstance(val, dict) and val.get("__frozenset__") is not None: return frozenset([ _unpack_value(item, whitelist_map) for item in val["__frozenset__"] ]) if isinstance(val, dict): return { key: _unpack_value(value, whitelist_map) for key, value in val.items() } return val
def def_from_pointer(pointer): target = pointer.load_target() if not callable(target): return _check_is_loadable(target) # if its a function invoke it - otherwise we are pointing to a # artifact in module scope, likely decorator output if seven.get_args(target): raise DagsterInvariantViolationError( "Error invoking function at {target} with no arguments. " "Reconstructable target must be callable with no arguments".format( target=pointer.describe())) return _check_is_loadable(target())