def __init__(self, name, input_defs, output_defs, description=None, metadata=None): self._name = check_valid_name(name) self._description = check.opt_str_param(description, 'description') self._metadata = check.opt_dict_param(metadata, 'metadata', key_type=str) self._input_defs = frozenlist(input_defs) self._input_dict = frozendict( {input_def.name: input_def for input_def in input_defs}) check.invariant( len(self._input_defs) == len(self._input_dict), 'Duplicate input def names') self._output_defs = frozenlist(output_defs) self._output_dict = frozendict( {output_def.name: output_def for output_def in output_defs}) check.invariant( len(self._output_defs) == len(self._output_dict), 'Duplicate output def names')
def __init__( self, name, input_defs, output_defs, description=None, tags=None, positional_inputs=None, ): self._name = check_valid_name(name) self._description = check.opt_str_param(description, "description") self._tags = validate_tags(tags) self._input_defs = frozenlist(input_defs) self._input_dict = frozendict( {input_def.name: input_def for input_def in input_defs}) check.invariant( len(self._input_defs) == len(self._input_dict), "Duplicate input def names") self._output_defs = frozenlist(output_defs) self._output_dict = frozendict( {output_def.name: output_def for output_def in output_defs}) check.invariant( len(self._output_defs) == len(self._output_dict), "Duplicate output def names") check.opt_list_param(positional_inputs, "positional_inputs", str) self._positional_inputs = (positional_inputs if positional_inputs is not None else list( map(lambda inp: inp.name, input_defs)))
def __new__(cls, reconstructor_pointer, reconstructable_args, reconstructable_kwargs): check.inst_param(reconstructor_pointer, "reconstructor_pointer", ModuleCodePointer) # These are lists rather than tuples to circumvent the tuple serdes machinery -- since these # are user-provided, they aren't whitelisted for serdes. check.list_param(reconstructable_args, "reconstructable_args") check.list_param(reconstructable_kwargs, "reconstructable_kwargs") for reconstructable_kwarg in reconstructable_kwargs: check.list_param(reconstructable_kwarg, "reconstructable_kwarg") check.invariant(check.is_str(reconstructable_kwarg[0]), "Bad kwarg key") check.invariant( len(reconstructable_kwarg) == 2, "Bad kwarg of length {length}, should be 2".format( length=len(reconstructable_kwarg)), ) # These are frozenlists, rather than lists, so that they can be hashed and the pointer # stored in the lru_cache on the repository and pipeline get_definition methods reconstructable_args = frozenlist(reconstructable_args) reconstructable_kwargs = frozenlist([ frozenlist(reconstructable_kwarg) for reconstructable_kwarg in reconstructable_kwargs ]) return super(CustomPointer, cls).__new__( cls, reconstructor_pointer, reconstructable_args, reconstructable_kwargs, )
def test_hash_frozen_list(): assert hash(frozenlist([])) assert hash(frozenlist(["foo", "bar"])) with pytest.raises(TypeError, match="unhashable type"): hash(frozenlist([[]])) with pytest.raises(TypeError, match="unhashable type"): hash(frozenlist([{}]))
def __new__( cls, repository_symbols, executable_path=None, repository_code_pointer_dict=None, entry_point=None, ): return super(ListRepositoriesResponse, cls).__new__( cls, repository_symbols=check.list_param( repository_symbols, "repository_symbols", of_type=LoadableRepositorySymbol), executable_path=check.opt_str_param(executable_path, "executable_path"), repository_code_pointer_dict=check.opt_dict_param( repository_code_pointer_dict, "repository_code_pointer_dict", key_type=str, value_type=CodePointer, ), entry_point=(frozenlist( check.list_param(entry_point, "entry_point", of_type=str)) if entry_point != None else None), )
def _recurse_in_to_array(context: TraversalContext, config_value: Any) -> EvaluateValueResult: check.invariant(context.config_type.kind == ConfigTypeKind.ARRAY, "Unexpected non array type") if not config_value: return EvaluateValueResult.for_value([]) if context.config_type.inner_type.kind != ConfigTypeKind.NONEABLE: if any((cv is None for cv in config_value)): check.failed("Null array member not caught in validation") results = [ _recursively_process_config(context.for_array(idx), item) for idx, item in enumerate(config_value) ] errors = [] for result in results: if not result.success: errors += result.errors if errors: return EvaluateValueResult.for_errors(errors) return EvaluateValueResult.for_value( frozenlist([result.value for result in results]))
def test_list_param(): assert check.list_param([], "list_param") == [] assert check.list_param(frozenlist(), "list_param") == [] with pytest.raises(ParameterCheckError): check.list_param(None, "list_param") with pytest.raises(ParameterCheckError): check.list_param("3u4", "list_param")
def test_list_param(): assert check.list_param([], 'list_param') == [] assert check.list_param(frozenlist(), 'list_param') == [] with pytest.raises(ParameterCheckError): check.list_param(None, 'list_param') with pytest.raises(ParameterCheckError): check.list_param('3u4', 'list_param')
def __init__(self, name, input_defs, output_defs, description=None, metadata=None): self.name = check_valid_name(name) self.description = check.opt_str_param(description, 'description') self.metadata = check.opt_dict_param(metadata, 'metadata', key_type=str) self.input_defs = frozenlist(input_defs) self.input_dict = frozendict( {input_def.name: input_def for input_def in input_defs}) self.output_defs = frozenlist(output_defs) self.output_dict = frozendict( {output_def.name: output_def for output_def in output_defs})
def test_list_param(): assert check.list_param([], "list_param") == [] assert check.list_param(frozenlist(), "list_param") == [] assert check.list_param(["foo"], "list_param", of_type=str) == ["foo"] with pytest.raises(ParameterCheckError): check.list_param(None, "list_param") with pytest.raises(ParameterCheckError): check.list_param("3u4", "list_param") with pytest.raises(CheckError): check.list_param(["foo"], "list_param", of_type=int)
def __new__(cls, executable_path, code_pointer, container_image=None, entry_point=None): return super(RepositoryPythonOrigin, cls).__new__( cls, check.str_param(executable_path, "executable_path"), check.inst_param(code_pointer, "code_pointer", CodePointer), check.opt_str_param(container_image, "container_image"), (frozenlist( check.list_param(entry_point, "entry_point", of_type=str)) if entry_point != None else None), )
def test_opt_nullable_list_param(): assert check.opt_nullable_list_param(None, "list_param") is None assert check.opt_nullable_list_param([], "list_param") == [] assert check.opt_nullable_list_param(frozenlist(), "list_param") == [] obj_list = [1] assert check.opt_nullable_list_param(obj_list, "list_param") == obj_list with pytest.raises(ParameterCheckError): check.opt_nullable_list_param(0, "list_param") with pytest.raises(ParameterCheckError): check.opt_nullable_list_param("", "list_param") with pytest.raises(ParameterCheckError): check.opt_nullable_list_param("3u4", "list_param")
def test_opt_nullable_list_param(): assert check.opt_nullable_list_param(None, 'list_param') is None assert check.opt_nullable_list_param([], 'list_param') == [] assert check.opt_nullable_list_param(frozenlist(), 'list_param') == [] obj_list = [1] assert check.opt_nullable_list_param(obj_list, 'list_param') == obj_list with pytest.raises(ParameterCheckError): check.opt_nullable_list_param(0, 'list_param') with pytest.raises(ParameterCheckError): check.opt_nullable_list_param('', 'list_param') with pytest.raises(ParameterCheckError): check.opt_nullable_list_param('3u4', 'list_param')
def __new__( cls, pointer, container_image=None, executable_path=None, entry_point=None, ): return super(ReconstructableRepository, cls).__new__( cls, pointer=check.inst_param(pointer, "pointer", CodePointer), container_image=check.opt_str_param(container_image, "container_image"), executable_path=check.opt_str_param(executable_path, "executable_path"), entry_point=( frozenlist(check.list_param(entry_point, "entry_point", of_type=str)) if entry_point != None else DEFAULT_DAGSTER_ENTRY_POINT ), )
def get_python_environment_entry_point(executable_path: str) -> List[str]: return frozenlist([executable_path, "-m", "dagster"])
def test_pickle_frozenlist(): orig_list = frozenlist([1, "a", {}]) data = pickle.dumps(orig_list) loaded_list = pickle.loads(data) assert orig_list == loaded_list
from typing import Any, Dict, List, NamedTuple, Optional from dagster import check from dagster.core.code_pointer import CodePointer from dagster.serdes import create_snapshot_id, whitelist_for_serdes from dagster.utils import frozenlist DEFAULT_DAGSTER_ENTRY_POINT = frozenlist(["dagster"]) def get_python_environment_entry_point(executable_path: str) -> List[str]: return frozenlist([executable_path, "-m", "dagster"]) @whitelist_for_serdes class RepositoryPythonOrigin( NamedTuple( "_RepositoryPythonOrigin", [ ("executable_path", str), ("code_pointer", CodePointer), ("container_image", Optional[str]), ("entry_point", Optional[List[str]]), ("container_context", Optional[Dict[str, Any]]), ], ), ): """ Args: executable_path (str): The Python executable of the user process. code_pointer (CodePoitner): Once the process has started, an object that can be used to find and load the repository in code.
def __init__( self, server_termination_event, loadable_target_origin=None, heartbeat=False, heartbeat_timeout=30, lazy_load_user_code=False, fixed_server_id=None, entry_point=None, ): super(DagsterApiServer, self).__init__() check.bool_param(heartbeat, "heartbeat") check.int_param(heartbeat_timeout, "heartbeat_timeout") check.invariant(heartbeat_timeout > 0, "heartbeat_timeout must be greater than 0") self._server_termination_event = check.inst_param( server_termination_event, "server_termination_event", ThreadingEventType ) self._loadable_target_origin = check.opt_inst_param( loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin ) self._mp_ctx = multiprocessing.get_context("spawn") # Each server is initialized with a unique UUID. This UUID is used by clients to track when # servers are replaced and is used for cache invalidation and reloading. self._server_id = check.opt_str_param(fixed_server_id, "fixed_server_id", str(uuid.uuid4())) # Client tells the server to shutdown by calling ShutdownServer (or by failing to send a # hearbeat, at which point this event is set. The cleanup thread will then set the server # termination event once all current executions have finished, which will stop the server) self._shutdown_once_executions_finish_event = threading.Event() # Dict[str, (multiprocessing.Process, DagsterInstance)] self._executions = {} # Dict[str, multiprocessing.Event] self._termination_events = {} self._termination_times = {} self._execution_lock = threading.Lock() self._serializable_load_error = None self._entry_point = ( frozenlist(check.list_param(entry_point, "entry_point", of_type=str)) if entry_point != None else DEFAULT_DAGSTER_ENTRY_POINT ) try: self._loaded_repositories = LoadedRepositories( loadable_target_origin, self._entry_point ) except Exception: if not lazy_load_user_code: raise self._loaded_repositories = None self._serializable_load_error = serializable_error_info_from_exc_info(sys.exc_info()) self.__last_heartbeat_time = time.time() if heartbeat: self.__heartbeat_thread = threading.Thread( target=self._heartbeat_thread, args=(heartbeat_timeout,), name="grpc-server-heartbeat", ) self.__heartbeat_thread.daemon = True self.__heartbeat_thread.start() else: self.__heartbeat_thread = None self.__cleanup_thread = threading.Thread( target=self._cleanup_thread, args=(), name="grpc-server-cleanup" ) self.__cleanup_thread.daemon = True self.__cleanup_thread.start()