def _mgr_fn(recon_repo):
"""Goes out of process but same process as host process"""
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
with Workspace(
[
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "setup.py"),
attribute="test_repo",
),
location_name="test",
),
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "setup.py"),
attribute="empty_repo",
),
location_name="empty_repo",
),
]
) as workspace:
yield workspace
def get_example_repository_location_handle():
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "repo.py"),
)
location_name = "example_repo_location"
origin = ManagedGrpcPythonEnvRepositoryLocationOrigin(loadable_target_origin, location_name)
return origin.create_handle()
def get_bar_repo_repository_location():
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "api_tests_repo.py"),
attribute="bar_repo",
)
location_name = "bar_repo_location"
origin = ManagedGrpcPythonEnvRepositoryLocationOrigin(loadable_target_origin, location_name)
with origin.create_test_location() as location:
yield location
def get_example_repository_location():
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
)
location_name = "example_repo_location"
origin = ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin, location_name)
with origin.create_test_location() as location:
yield location
def test_bad_load():
with _default_instance() as instance:
instance = DagsterInstance.get()
working_directory = os.path.dirname(__file__)
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
attribute="doesnt_exist",
working_directory=working_directory,
)
repo_origin = ExternalRepositoryOrigin(
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=loadable_target_origin
),
"doesnt_exist",
)
schedule_origin = repo_origin.get_job_origin("also_doesnt_exist")
result = sync_launch_scheduled_execution(schedule_origin)
assert isinstance(result, ScheduledExecutionFailed)
assert "doesnt_exist not found at module scope in file" in result.errors[0].to_string()
ticks = instance.get_job_ticks(schedule_origin.get_id())
assert ticks[0].status == JobTickStatus.FAILURE
assert "doesnt_exist not found at module scope in file" in ticks[0].error.message
def test_origin_ids_stable():
# This test asserts fixed schedule origin IDs to prevent any changes from
# accidentally shifting these ids that are persisted to ScheduleStorage
python_origin = ExternalJobOrigin(
ExternalRepositoryOrigin(
ManagedGrpcPythonEnvRepositoryLocationOrigin(
LoadableTargetOrigin(
executable_path="/fake/executable",
python_file="/fake/file/path",
attribute="fake_attribute",
)
),
"fake_repo",
),
"fake_schedule",
)
assert python_origin.get_id() == "eb01cc697463ba614a67567fdeaafcccc60f0fc4"
grpc_origin = ExternalJobOrigin(
ExternalRepositoryOrigin(
GrpcServerRepositoryLocationOrigin(host="fakehost", port=52618), "repo_name"
),
"fake_schedule",
)
assert grpc_origin.get_id() == "0961ecddbddfc71104adf036ebe8cd97a94dc77b"
def fake_repo_target():
return ExternalRepositoryOrigin(
ManagedGrpcPythonEnvRepositoryLocationOrigin(
LoadableTargetOrigin(executable_path=sys.executable,
module_name="fake",
attribute="fake"), ),
"fake_repo_name",
)
def get_bar_repo_grpc_repository_location_handle():
with ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
attribute="bar_repo",
python_file=file_relative_path(__file__, "api_tests_repo.py"),
),
location_name="bar_repo",
).create_handle() as handle:
yield handle
def test_dagster_out_of_process_location():
with ManagedGrpcPythonEnvRepositoryLocationOrigin(
location_name="test_location",
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "setup.py"),
attribute="test_repo",
),
).create_test_location() as env:
assert env.get_repository("test_repo")
def get_external_pipeline_from_managed_grpc_python_env_repository(pipeline_name):
with ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
attribute="nope",
python_file=file_relative_path(__file__, "test_default_run_launcher.py"),
),
location_name="nope",
).create_test_location() as repository_location:
yield repository_location.get_repository("nope").get_full_external_pipeline(pipeline_name)
def get_test_external_repo():
with ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
attribute="test_repository",
),
location_name="test_location",
).create_handle() as handle:
yield handle.create_location().get_repository("test_repository")
def test_terminate_after_shutdown():
with instance_for_test() as instance:
origin = ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
attribute="nope",
python_file=file_relative_path(__file__,
"test_default_run_launcher.py"),
),
location_name="nope",
)
with origin.create_test_location() as repository_location:
external_pipeline = repository_location.get_repository(
"nope").get_full_external_pipeline("sleepy_pipeline")
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=sleepy_pipeline, run_config=None)
instance.launch_run(pipeline_run.run_id, external_pipeline)
poll_for_step_start(instance, pipeline_run.run_id)
# Tell the server to shut down once executions finish
repository_location.grpc_server_registry.get_grpc_endpoint(
origin).create_client().shutdown_server()
# Trying to start another run fails
doomed_to_fail_external_pipeline = repository_location.get_repository(
"nope").get_full_external_pipeline("math_diamond")
doomed_to_fail_pipeline_run = instance.create_run_for_pipeline(
pipeline_def=math_diamond, run_config=None)
with pytest.raises(DagsterLaunchFailedError):
instance.launch_run(doomed_to_fail_pipeline_run.run_id,
doomed_to_fail_external_pipeline)
launcher = instance.run_launcher
# Can terminate the run even after the shutdown event has been received
assert launcher.can_terminate(pipeline_run.run_id)
assert launcher.terminate(pipeline_run.run_id)
def get_giant_repo_grpc_repository_location_handle():
with ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
attribute="giant_repo",
module_name=
"dagster_tests.api_tests.test_api_snapshot_repository",
),
location_name="giant_repo_location",
).create_handle() as handle:
yield handle
def default_repo():
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable, python_file=__file__, working_directory=os.getcwd(),
)
with RepositoryLocationHandle.create_from_repository_location_origin(
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=loadable_target_origin, location_name="test_location",
)
) as handle:
yield RepositoryLocation.from_handle(handle).get_repository("the_repo")
def define_out_of_process_workspace(python_file, fn_name):
return Workspace([
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=python_file,
attribute=fn_name,
),
location_name=main_repo_location_name(),
)
])
def get_bar_repo_repository_location_handle():
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "api_tests_repo.py"),
)
location_name = "bar_repo_location"
origin = ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin, location_name)
return RepositoryLocationHandle.create_from_repository_location_origin(
origin)
def python_schedule_origin(schedule_name):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable, python_file=__file__, attribute="the_repo"
)
repo_origin = ExternalRepositoryOrigin(
ManagedGrpcPythonEnvRepositoryLocationOrigin(loadable_target_origin=loadable_target_origin),
"the_repo",
)
yield repo_origin.get_job_origin(schedule_name)
def default_repo():
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
working_directory=os.getcwd(),
)
with ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=loadable_target_origin,
location_name="test_location",
).create_test_location() as location:
yield location.get_repository("the_repo")
def _mgr_fn(recon_repo):
"""Goes out of process via grpc"""
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
loadable_target_origin = recon_repo.get_origin().loadable_target_origin
with Workspace(
[
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=loadable_target_origin, location_name="test",
)
]
) as workspace:
yield workspace
def define_out_of_process_context(python_file, fn_name, instance):
check.inst_param(instance, "instance", DagsterInstance)
with Workspace([
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=python_file,
attribute=fn_name,
),
location_name=main_repo_location_name(),
)
]) as workspace:
yield DagsterGraphQLContext(
workspace=workspace,
instance=instance,
)
def test_terminate_after_shutdown():
with instance_for_test() as instance:
with RepositoryLocationHandle.create_from_repository_location_origin(
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
attribute="nope",
python_file=file_relative_path(
__file__, "test_default_run_launcher.py"),
),
location_name="nope",
)) as repository_location_handle:
repository_location = GrpcServerRepositoryLocation(
repository_location_handle)
external_pipeline = repository_location.get_repository(
"nope").get_full_external_pipeline("sleepy_pipeline")
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=sleepy_pipeline, run_config=None)
instance.launch_run(pipeline_run.run_id, external_pipeline)
poll_for_step_start(instance, pipeline_run.run_id)
# Leaving this context manager cleans up the repository location handle,
# which tells the server to shut down once executions finish
# Trying to start another run fails
doomed_to_fail_external_pipeline = repository_location.get_repository(
"nope").get_full_external_pipeline("math_diamond")
doomed_to_fail_pipeline_run = instance.create_run_for_pipeline(
pipeline_def=math_diamond, run_config=None)
with pytest.raises(DagsterLaunchFailedError):
instance.launch_run(doomed_to_fail_pipeline_run.run_id,
doomed_to_fail_external_pipeline)
launcher = instance.run_launcher
# Can terminate the run even after the shutdown event has been received
assert launcher.can_terminate(pipeline_run.run_id)
assert launcher.terminate(pipeline_run.run_id)
def test_origin_id():
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
attribute="the_repo")
location_name = "the_location"
origin = ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin, location_name)
with RepositoryLocationHandle.create_from_repository_location_origin(
origin) as handle:
host_location = RepositoryLocation.from_handle(handle)
external_pipeline = host_location.get_repository(
"the_repo").get_full_external_pipeline("the_pipe")
recon_pipeline = recon_pipeline_from_origin(
external_pipeline.get_origin())
assert external_pipeline.get_origin_id(
) == recon_pipeline.get_origin_id()
def _create_python_env_location_origin(loadable_target_origin, location_name):
return ManagedGrpcPythonEnvRepositoryLocationOrigin(loadable_target_origin, location_name)
def test_load_workspace(self, graphql_context):
# Add an error origin
original_origins = location_origins_from_yaml_paths(
[file_relative_path(__file__, "multi_location.yaml")])
with mock.patch(
"dagster.core.workspace.load_target.location_origins_from_yaml_paths",
) as origins_mock:
original_origins.append(
ManagedGrpcPythonEnvRepositoryLocationOrigin(
location_name="error_location",
loadable_target_origin=LoadableTargetOrigin(
python_file="made_up_file.py",
executable_path=sys.executable),
))
origins_mock.return_value = original_origins
reload_time = time.time()
new_context = graphql_context.reload_workspace()
result = execute_dagster_graphql(new_context, WORKSPACE_QUERY)
assert result
assert result.data
assert result.data["workspaceOrError"]
assert result.data["workspaceOrError"][
"__typename"] == "Workspace", str(result.data)
nodes = result.data["workspaceOrError"]["locationEntries"]
assert len(nodes) == 3
assert all([
node["__typename"] == "WorkspaceLocationEntry"
for node in nodes
]), str(nodes)
success_nodes = [
node for node in nodes if node["locationOrLoadError"]
["__typename"] == "RepositoryLocation"
]
assert len(success_nodes) == 2
failures = [
node for node in nodes
if node["locationOrLoadError"]["__typename"] == "PythonError"
]
assert len(failures) == 1
failure_node = failures[0]
assert failure_node["name"] == "error_location"
assert failure_node["loadStatus"] == "LOADED"
assert "No such file or directory" in failure_node[
"locationOrLoadError"]["message"]
for node in nodes:
assert node["loadStatus"] == "LOADED"
update_time = node["updatedTimestamp"]
assert update_time >= reload_time and update_time <= time.time(
)
metadatas = node["displayMetadata"]
metadata_dict = {
metadata["key"]: metadata["value"]
for metadata in metadatas
}
assert ("python_file" in metadata_dict
or "module_name" in metadata_dict
or "package_name" in metadata_dict)
def test_reload_workspace(self, graphql_context):
result = execute_dagster_graphql(graphql_context, RELOAD_WORKSPACE_QUERY)
assert result
assert result.data
assert result.data["reloadWorkspace"]
assert result.data["reloadWorkspace"]["__typename"] == "RepositoryLocationConnection"
nodes = result.data["reloadWorkspace"]["nodes"]
assert len(nodes) == 2
assert all([node["__typename"] == "RepositoryLocation" for node in nodes])
original_origins = location_origins_from_yaml_paths(
[file_relative_path(__file__, "multi_location.yaml")]
)
# simulate removing all the origins
with mock.patch(
"dagster.cli.workspace.cli_target.location_origins_from_yaml_paths",
) as origins_mock:
# simulate removing an origin, reload
origins_mock.return_value = original_origins[0:1]
result = execute_dagster_graphql(graphql_context, RELOAD_WORKSPACE_QUERY)
assert result
assert result.data
assert result.data["reloadWorkspace"]
assert result.data["reloadWorkspace"]["__typename"] == "RepositoryLocationConnection"
nodes = result.data["reloadWorkspace"]["nodes"]
assert len(nodes) == 1
assert all([node["__typename"] == "RepositoryLocation" for node in nodes])
# Simulate adding an origin with an error, reload
original_origins.append(
ManagedGrpcPythonEnvRepositoryLocationOrigin(
location_name="error_location",
loadable_target_origin=LoadableTargetOrigin(
python_file="made_up_file.py", executable_path=sys.executable
),
)
)
origins_mock.return_value = original_origins
result = execute_dagster_graphql(graphql_context, RELOAD_WORKSPACE_QUERY)
assert result
assert result.data
assert result.data["reloadWorkspace"]
assert result.data["reloadWorkspace"]["__typename"] == "RepositoryLocationConnection"
nodes = result.data["reloadWorkspace"]["nodes"]
assert len(nodes) == 3
assert len([node for node in nodes if node["__typename"] == "RepositoryLocation"]) == 2
failures = [
node for node in nodes if node["__typename"] == "RepositoryLocationLoadFailure"
]
assert len(failures) == 1
assert failures[0]["name"] == "error_location"
# Add another origin without an error, reload
original_origins.append(original_origins[0]._replace(location_name="location_copy"))
origins_mock.return_value = original_origins
result = execute_dagster_graphql(graphql_context, RELOAD_WORKSPACE_QUERY)
nodes = result.data["reloadWorkspace"]["nodes"]
assert len(nodes) == 4
assert len([node for node in nodes if node["__typename"] == "RepositoryLocation"]) == 3
failures = [
node for node in nodes if node["__typename"] == "RepositoryLocationLoadFailure"
]
assert len(failures) == 1
assert "location_copy" in [node["name"] for node in nodes]
assert original_origins[0].location_name in [node["name"] for node in nodes]
# Finally, update one of the origins' location names
original_origins[0] = original_origins[0]._replace(location_name="new_location_name")
result = execute_dagster_graphql(graphql_context, RELOAD_WORKSPACE_QUERY)
nodes = result.data["reloadWorkspace"]["nodes"]
assert len(nodes) == 4
assert len([node for node in nodes if node["__typename"] == "RepositoryLocation"]) == 3
failures = [
node for node in nodes if node["__typename"] == "RepositoryLocationLoadFailure"
]
assert len(failures) == 1
assert "new_location_name" in [node["name"] for node in nodes]
def default_repo():
with ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=loadable_target_origin(),
location_name="test_location",
).create_test_location() as location:
yield location.get_repository("the_repo")