def __init__(self,
name,
input_defs,
output_defs,
description=None,
metadata=None):
self._name = check_valid_name(name)
self._description = check.opt_str_param(description, 'description')
self._metadata = check.opt_dict_param(metadata,
'metadata',
key_type=str)
self._input_defs = frozenlist(input_defs)
self._input_dict = frozendict(
{input_def.name: input_def
for input_def in input_defs})
check.invariant(
len(self._input_defs) == len(self._input_dict),
'Duplicate input def names')
self._output_defs = frozenlist(output_defs)
self._output_dict = frozendict(
{output_def.name: output_def
for output_def in output_defs})
check.invariant(
len(self._output_defs) == len(self._output_dict),
'Duplicate output def names')
def __init__(
self,
name,
input_defs,
output_defs,
description=None,
tags=None,
positional_inputs=None,
):
self._name = check_valid_name(name)
self._description = check.opt_str_param(description, "description")
self._tags = validate_tags(tags)
self._input_defs = frozenlist(input_defs)
self._input_dict = frozendict(
{input_def.name: input_def
for input_def in input_defs})
check.invariant(
len(self._input_defs) == len(self._input_dict),
"Duplicate input def names")
self._output_defs = frozenlist(output_defs)
self._output_dict = frozendict(
{output_def.name: output_def
for output_def in output_defs})
check.invariant(
len(self._output_defs) == len(self._output_dict),
"Duplicate output def names")
check.opt_list_param(positional_inputs, "positional_inputs", str)
self._positional_inputs = (positional_inputs
if positional_inputs is not None else list(
map(lambda inp: inp.name, input_defs)))
def __new__(cls, reconstructor_pointer, reconstructable_args,
reconstructable_kwargs):
check.inst_param(reconstructor_pointer, "reconstructor_pointer",
ModuleCodePointer)
# These are lists rather than tuples to circumvent the tuple serdes machinery -- since these
# are user-provided, they aren't whitelisted for serdes.
check.list_param(reconstructable_args, "reconstructable_args")
check.list_param(reconstructable_kwargs, "reconstructable_kwargs")
for reconstructable_kwarg in reconstructable_kwargs:
check.list_param(reconstructable_kwarg, "reconstructable_kwarg")
check.invariant(check.is_str(reconstructable_kwarg[0]),
"Bad kwarg key")
check.invariant(
len(reconstructable_kwarg) == 2,
"Bad kwarg of length {length}, should be 2".format(
length=len(reconstructable_kwarg)),
)
# These are frozenlists, rather than lists, so that they can be hashed and the pointer
# stored in the lru_cache on the repository and pipeline get_definition methods
reconstructable_args = frozenlist(reconstructable_args)
reconstructable_kwargs = frozenlist([
frozenlist(reconstructable_kwarg)
for reconstructable_kwarg in reconstructable_kwargs
])
return super(CustomPointer, cls).__new__(
cls,
reconstructor_pointer,
reconstructable_args,
reconstructable_kwargs,
)
def test_hash_frozen_list():
assert hash(frozenlist([]))
assert hash(frozenlist(["foo", "bar"]))
with pytest.raises(TypeError, match="unhashable type"):
hash(frozenlist([[]]))
with pytest.raises(TypeError, match="unhashable type"):
hash(frozenlist([{}]))
def __new__(
cls,
repository_symbols,
executable_path=None,
repository_code_pointer_dict=None,
entry_point=None,
):
return super(ListRepositoriesResponse, cls).__new__(
cls,
repository_symbols=check.list_param(
repository_symbols,
"repository_symbols",
of_type=LoadableRepositorySymbol),
executable_path=check.opt_str_param(executable_path,
"executable_path"),
repository_code_pointer_dict=check.opt_dict_param(
repository_code_pointer_dict,
"repository_code_pointer_dict",
key_type=str,
value_type=CodePointer,
),
entry_point=(frozenlist(
check.list_param(entry_point, "entry_point", of_type=str))
if entry_point != None else None),
)
def _recurse_in_to_array(context: TraversalContext,
config_value: Any) -> EvaluateValueResult:
check.invariant(context.config_type.kind == ConfigTypeKind.ARRAY,
"Unexpected non array type")
if not config_value:
return EvaluateValueResult.for_value([])
if context.config_type.inner_type.kind != ConfigTypeKind.NONEABLE:
if any((cv is None for cv in config_value)):
check.failed("Null array member not caught in validation")
results = [
_recursively_process_config(context.for_array(idx), item)
for idx, item in enumerate(config_value)
]
errors = []
for result in results:
if not result.success:
errors += result.errors
if errors:
return EvaluateValueResult.for_errors(errors)
return EvaluateValueResult.for_value(
frozenlist([result.value for result in results]))
def test_list_param():
assert check.list_param([], "list_param") == []
assert check.list_param(frozenlist(), "list_param") == []
with pytest.raises(ParameterCheckError):
check.list_param(None, "list_param")
with pytest.raises(ParameterCheckError):
check.list_param("3u4", "list_param")
def test_list_param():
assert check.list_param([], 'list_param') == []
assert check.list_param(frozenlist(), 'list_param') == []
with pytest.raises(ParameterCheckError):
check.list_param(None, 'list_param')
with pytest.raises(ParameterCheckError):
check.list_param('3u4', 'list_param')
def __init__(self,
name,
input_defs,
output_defs,
description=None,
metadata=None):
self.name = check_valid_name(name)
self.description = check.opt_str_param(description, 'description')
self.metadata = check.opt_dict_param(metadata,
'metadata',
key_type=str)
self.input_defs = frozenlist(input_defs)
self.input_dict = frozendict(
{input_def.name: input_def
for input_def in input_defs})
self.output_defs = frozenlist(output_defs)
self.output_dict = frozendict(
{output_def.name: output_def
for output_def in output_defs})
def test_list_param():
assert check.list_param([], "list_param") == []
assert check.list_param(frozenlist(), "list_param") == []
assert check.list_param(["foo"], "list_param", of_type=str) == ["foo"]
with pytest.raises(ParameterCheckError):
check.list_param(None, "list_param")
with pytest.raises(ParameterCheckError):
check.list_param("3u4", "list_param")
with pytest.raises(CheckError):
check.list_param(["foo"], "list_param", of_type=int)
def __new__(cls,
executable_path,
code_pointer,
container_image=None,
entry_point=None):
return super(RepositoryPythonOrigin, cls).__new__(
cls,
check.str_param(executable_path, "executable_path"),
check.inst_param(code_pointer, "code_pointer", CodePointer),
check.opt_str_param(container_image, "container_image"),
(frozenlist(
check.list_param(entry_point, "entry_point", of_type=str))
if entry_point != None else None),
)
def test_opt_nullable_list_param():
assert check.opt_nullable_list_param(None, "list_param") is None
assert check.opt_nullable_list_param([], "list_param") == []
assert check.opt_nullable_list_param(frozenlist(), "list_param") == []
obj_list = [1]
assert check.opt_nullable_list_param(obj_list, "list_param") == obj_list
with pytest.raises(ParameterCheckError):
check.opt_nullable_list_param(0, "list_param")
with pytest.raises(ParameterCheckError):
check.opt_nullable_list_param("", "list_param")
with pytest.raises(ParameterCheckError):
check.opt_nullable_list_param("3u4", "list_param")
def test_opt_nullable_list_param():
assert check.opt_nullable_list_param(None, 'list_param') is None
assert check.opt_nullable_list_param([], 'list_param') == []
assert check.opt_nullable_list_param(frozenlist(), 'list_param') == []
obj_list = [1]
assert check.opt_nullable_list_param(obj_list, 'list_param') == obj_list
with pytest.raises(ParameterCheckError):
check.opt_nullable_list_param(0, 'list_param')
with pytest.raises(ParameterCheckError):
check.opt_nullable_list_param('', 'list_param')
with pytest.raises(ParameterCheckError):
check.opt_nullable_list_param('3u4', 'list_param')
def __new__(
cls,
pointer,
container_image=None,
executable_path=None,
entry_point=None,
):
return super(ReconstructableRepository, cls).__new__(
cls,
pointer=check.inst_param(pointer, "pointer", CodePointer),
container_image=check.opt_str_param(container_image, "container_image"),
executable_path=check.opt_str_param(executable_path, "executable_path"),
entry_point=(
frozenlist(check.list_param(entry_point, "entry_point", of_type=str))
if entry_point != None
else DEFAULT_DAGSTER_ENTRY_POINT
),
)
def get_python_environment_entry_point(executable_path: str) -> List[str]:
return frozenlist([executable_path, "-m", "dagster"])
def test_pickle_frozenlist():
orig_list = frozenlist([1, "a", {}])
data = pickle.dumps(orig_list)
loaded_list = pickle.loads(data)
assert orig_list == loaded_list
from typing import Any, Dict, List, NamedTuple, Optional
from dagster import check
from dagster.core.code_pointer import CodePointer
from dagster.serdes import create_snapshot_id, whitelist_for_serdes
from dagster.utils import frozenlist
DEFAULT_DAGSTER_ENTRY_POINT = frozenlist(["dagster"])
def get_python_environment_entry_point(executable_path: str) -> List[str]:
return frozenlist([executable_path, "-m", "dagster"])
@whitelist_for_serdes
class RepositoryPythonOrigin(
NamedTuple(
"_RepositoryPythonOrigin",
[
("executable_path", str),
("code_pointer", CodePointer),
("container_image", Optional[str]),
("entry_point", Optional[List[str]]),
("container_context", Optional[Dict[str, Any]]),
],
), ):
"""
Args:
executable_path (str): The Python executable of the user process.
code_pointer (CodePoitner): Once the process has started, an object that can be used to
find and load the repository in code.
def __init__(
self,
server_termination_event,
loadable_target_origin=None,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
fixed_server_id=None,
entry_point=None,
):
super(DagsterApiServer, self).__init__()
check.bool_param(heartbeat, "heartbeat")
check.int_param(heartbeat_timeout, "heartbeat_timeout")
check.invariant(heartbeat_timeout > 0, "heartbeat_timeout must be greater than 0")
self._server_termination_event = check.inst_param(
server_termination_event, "server_termination_event", ThreadingEventType
)
self._loadable_target_origin = check.opt_inst_param(
loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin
)
self._mp_ctx = multiprocessing.get_context("spawn")
# Each server is initialized with a unique UUID. This UUID is used by clients to track when
# servers are replaced and is used for cache invalidation and reloading.
self._server_id = check.opt_str_param(fixed_server_id, "fixed_server_id", str(uuid.uuid4()))
# Client tells the server to shutdown by calling ShutdownServer (or by failing to send a
# hearbeat, at which point this event is set. The cleanup thread will then set the server
# termination event once all current executions have finished, which will stop the server)
self._shutdown_once_executions_finish_event = threading.Event()
# Dict[str, (multiprocessing.Process, DagsterInstance)]
self._executions = {}
# Dict[str, multiprocessing.Event]
self._termination_events = {}
self._termination_times = {}
self._execution_lock = threading.Lock()
self._serializable_load_error = None
self._entry_point = (
frozenlist(check.list_param(entry_point, "entry_point", of_type=str))
if entry_point != None
else DEFAULT_DAGSTER_ENTRY_POINT
)
try:
self._loaded_repositories = LoadedRepositories(
loadable_target_origin, self._entry_point
)
except Exception:
if not lazy_load_user_code:
raise
self._loaded_repositories = None
self._serializable_load_error = serializable_error_info_from_exc_info(sys.exc_info())
self.__last_heartbeat_time = time.time()
if heartbeat:
self.__heartbeat_thread = threading.Thread(
target=self._heartbeat_thread,
args=(heartbeat_timeout,),
name="grpc-server-heartbeat",
)
self.__heartbeat_thread.daemon = True
self.__heartbeat_thread.start()
else:
self.__heartbeat_thread = None
self.__cleanup_thread = threading.Thread(
target=self._cleanup_thread, args=(), name="grpc-server-cleanup"
)
self.__cleanup_thread.daemon = True
self.__cleanup_thread.start()
