def workspace_load_target():
return PythonFileTarget(
python_file=__file__,
attribute=None,
working_directory=os.path.dirname(__file__),
location_name="test_location",
)
def _mgr_fn(recon_repo, instance, read_only):
"""Goes out of process via grpc"""
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
loadable_target_origin = recon_repo.get_python_origin().loadable_target_origin
with WorkspaceProcessContext(
instance,
(
PythonFileTarget(
python_file=loadable_target_origin.python_file,
attribute=loadable_target_origin.attribute,
working_directory=loadable_target_origin.working_directory,
location_name="test",
)
if loadable_target_origin.python_file
else ModuleTarget(
module_name=loadable_target_origin.module_name,
attribute=loadable_target_origin.attribute,
location_name="test",
)
),
version="",
read_only=read_only,
) as workspace_process_context:
yield workspace_process_context
def test_terminate_after_shutdown():
with instance_for_test() as instance:
with WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=file_relative_path(__file__, "test_default_run_launcher.py"),
attribute="nope",
working_directory=None,
location_name="test",
),
) as workspace_process_context:
workspace = workspace_process_context.create_request_context()
external_pipeline = (
workspace.get_repository_location("test")
.get_repository("nope")
.get_full_external_pipeline("sleepy_pipeline")
)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=sleepy_pipeline,
run_config=None,
external_pipeline_origin=external_pipeline.get_external_origin(),
pipeline_code_origin=external_pipeline.get_python_origin(),
)
instance.launch_run(pipeline_run.run_id, workspace)
poll_for_step_start(instance, pipeline_run.run_id)
repository_location = workspace.get_repository_location("test")
# Tell the server to shut down once executions finish
repository_location.grpc_server_registry.get_grpc_endpoint(
repository_location.origin
).create_client().shutdown_server()
external_pipeline = (
workspace.get_repository_location("test")
.get_repository("nope")
.get_full_external_pipeline("math_diamond")
)
doomed_to_fail_pipeline_run = instance.create_run_for_pipeline(
pipeline_def=math_diamond,
run_config=None,
external_pipeline_origin=external_pipeline.get_external_origin(),
pipeline_code_origin=external_pipeline.get_python_origin(),
)
with pytest.raises(DagsterLaunchFailedError):
instance.launch_run(doomed_to_fail_pipeline_run.run_id, workspace)
launcher = instance.run_launcher
# Can terminate the run even after the shutdown event has been received
assert launcher.can_terminate(pipeline_run.run_id)
assert launcher.terminate(pipeline_run.run_id)
def get_main_workspace(instance):
with WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=file_relative_path(__file__, "setup.py"),
attribute=main_repo_name(),
working_directory=None,
location_name=main_repo_location_name(),
),
) as workspace_process_context:
yield workspace_process_context.create_request_context()
def define_out_of_process_workspace(python_file, fn_name, instance):
return WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=python_file,
attribute=fn_name,
working_directory=None,
location_name=main_repo_location_name(),
),
version="",
)
def foo_example_workspace_fixture(instance):
with WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=file_relative_path(__file__, "repo.py"),
attribute=None,
working_directory=os.path.dirname(__file__),
location_name=None,
),
) as workspace_process_context:
yield workspace_process_context.create_request_context()
def get_bar_workspace(instance):
with WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=file_relative_path(__file__, "api_tests_repo.py"),
attribute="bar_repo",
working_directory=None,
location_name="bar_repo_location",
),
) as workspace_process_context:
yield workspace_process_context.create_request_context()
def get_managed_grpc_server_workspace(instance):
with WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=file_relative_path(__file__, "test_default_run_launcher.py"),
attribute="nope",
working_directory=None,
location_name="test",
),
) as workspace_process_context:
yield workspace_process_context.create_request_context()
def _mgr_fn(instance, read_only):
"""Goes out of process but same process as host process"""
with WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=file_relative_path(__file__, "setup.py"),
attribute="test_dict_repo",
working_directory=None,
location_name="test",
),
version="",
read_only=read_only,
) as workspace:
yield workspace
def _mgr_fn(instance, read_only):
"""Goes out of process via grpc"""
loadable_target_origin = get_main_loadable_target_origin()
with WorkspaceProcessContext(
instance,
(PythonFileTarget(
python_file=loadable_target_origin.python_file,
attribute=loadable_target_origin.attribute,
working_directory=loadable_target_origin.working_directory,
location_name="test",
) if loadable_target_origin.python_file else ModuleTarget(
module_name=loadable_target_origin.module_name,
attribute=loadable_target_origin.attribute,
working_directory=loadable_target_origin.working_directory,
location_name="test",
)),
version="",
read_only=read_only,
) as workspace_process_context:
yield workspace_process_context
def created_workspace_load_target(kwargs):
check.dict_param(kwargs, "kwargs")
if are_all_keys_empty(kwargs, WORKSPACE_CLI_ARGS):
if kwargs.get("empty_workspace"):
return EmptyWorkspaceTarget()
if os.path.exists("workspace.yaml"):
return WorkspaceFileTarget(paths=["workspace.yaml"])
raise click.UsageError("No arguments given and workspace.yaml not found.")
if kwargs.get("workspace"):
_check_cli_arguments_none(
kwargs,
"python_file",
"working_directory",
"empty_working_directory",
"module_name",
"package_name",
"attribute",
"grpc_host",
"grpc_port",
"grpc_socket",
)
return WorkspaceFileTarget(paths=list(kwargs["workspace"]))
if kwargs.get("python_file"):
_check_cli_arguments_none(
kwargs,
"module_name",
"package_name",
"grpc_host",
"grpc_port",
"grpc_socket",
)
working_directory = get_working_directory_from_kwargs(kwargs)
return PythonFileTarget(
python_file=kwargs.get("python_file"),
attribute=kwargs.get("attribute"),
working_directory=working_directory,
location_name=None,
)
if kwargs.get("module_name"):
_check_cli_arguments_none(
kwargs,
"package_name",
"working_directory",
"empty_working_directory",
"grpc_host",
"grpc_port",
"grpc_socket",
)
return ModuleTarget(
module_name=kwargs.get("module_name"),
attribute=kwargs.get("attribute"),
location_name=None,
)
if kwargs.get("package_name"):
_check_cli_arguments_none(
kwargs,
"working_directory",
"empty_working_directory",
"grpc_host",
"grpc_port",
"grpc_socket",
)
return PackageTarget(
package_name=kwargs.get("package_name"),
attribute=kwargs.get("attribute"),
location_name=None,
)
if kwargs.get("grpc_port"):
_check_cli_arguments_none(
kwargs,
"attribute",
"working_directory",
"empty_working_directory",
"grpc_socket",
)
return GrpcServerTarget(
port=kwargs.get("grpc_port"),
socket=None,
host=(kwargs.get("grpc_host") if kwargs.get("grpc_host") else "localhost"),
location_name=None,
)
elif kwargs.get("grpc_socket"):
_check_cli_arguments_none(
kwargs,
"attribute",
"working_directory",
"empty_working_directory",
)
return GrpcServerTarget(
port=None,
socket=kwargs.get("grpc_socket"),
host=(kwargs.get("grpc_host") if kwargs.get("grpc_host") else "localhost"),
location_name=None,
)
else:
_cli_load_invariant(False)
def get_workspace_load_target(kwargs: Dict[str, str]):
check.dict_param(kwargs, "kwargs")
if are_all_keys_empty(kwargs, WORKSPACE_CLI_ARGS):
if kwargs.get("empty_workspace"):
return EmptyWorkspaceTarget()
if os.path.exists("workspace.yaml"):
return WorkspaceFileTarget(paths=["workspace.yaml"])
raise click.UsageError(
"No arguments given and workspace.yaml not found.")
if kwargs.get("workspace"):
_check_cli_arguments_none(
kwargs,
"python_file",
"working_directory",
"module_name",
"package_name",
"attribute",
"grpc_host",
"grpc_port",
"grpc_socket",
)
return WorkspaceFileTarget(
paths=list(cast(Union[List, Tuple], kwargs.get("workspace"))))
if kwargs.get("python_file"):
_check_cli_arguments_none(
kwargs,
"module_name",
"package_name",
"grpc_host",
"grpc_port",
"grpc_socket",
)
working_directory = get_working_directory_from_kwargs(kwargs)
return PythonFileTarget(
python_file=check.str_elem(kwargs, "python_file"),
attribute=check.opt_str_elem(kwargs, "attribute"),
working_directory=working_directory,
location_name=None,
)
if kwargs.get("module_name"):
_check_cli_arguments_none(
kwargs,
"package_name",
"grpc_host",
"grpc_port",
"grpc_socket",
)
working_directory = get_working_directory_from_kwargs(kwargs)
return ModuleTarget(
module_name=check.str_elem(kwargs, "module_name"),
attribute=check.opt_str_elem(kwargs, "attribute"),
working_directory=working_directory,
location_name=None,
)
if kwargs.get("package_name"):
_check_cli_arguments_none(
kwargs,
"grpc_host",
"grpc_port",
"grpc_socket",
)
working_directory = get_working_directory_from_kwargs(kwargs)
return PackageTarget(
package_name=check.str_elem(kwargs, "package_name"),
attribute=check.opt_str_elem(kwargs, "attribute"),
working_directory=working_directory,
location_name=None,
)
if kwargs.get("grpc_port"):
_check_cli_arguments_none(
kwargs,
"attribute",
"working_directory",
"grpc_socket",
)
return GrpcServerTarget(
port=check.int_elem(kwargs, "grpc_port"),
socket=None,
host=check.opt_str_elem(kwargs, "grpc_host") or "localhost",
location_name=None,
)
elif kwargs.get("grpc_socket"):
_check_cli_arguments_none(
kwargs,
"attribute",
"working_directory",
)
return GrpcServerTarget(
port=None,
socket=check.str_elem(kwargs, "grpc_socket"),
host=check.opt_str_elem(kwargs, "grpc_host") or "localhost",
location_name=None,
)
else:
_cli_load_invariant(False)
# necessary for pyright, does not understand _cli_load_invariant(False) never returns
assert False