def test_reload_on_request_context():
with instance_for_test() as instance:
with define_out_of_process_workspace(__file__,
"get_repo") as workspace:
# Create a process context
process_context = WorkspaceProcessContext(workspace=workspace,
instance=instance)
assert len(process_context.repository_locations) == 1
# Save the repository name
repository_location = process_context.repository_locations[0]
repo = list(repository_location.get_repositories().values())[0]
repo_name = repo.name
# Create a request context from the process context
request_context = process_context.create_request_context()
# Reload the location and save the new repository name
process_context.reload_repository_location(
repository_location.name)
repository_location = process_context.repository_locations[0]
repo = list(repository_location.get_repositories().values())[0]
new_repo_name = repo.name
# Save the repository name from the request context
repository_location = request_context.repository_locations[0]
repo = list(repository_location.get_repositories().values())[0]
request_context_repo_name = repo.name
# Check that the repository has changed
assert repo_name != new_repo_name
# Check that the repository name is still the same on the request context,
# confirming that the old repository location is still running
assert repo_name == request_context_repo_name
def test_can_reload_on_external_repository_error():
with instance_for_test() as instance:
with ExitStack() as exit_stack:
with mock.patch(
# note it where the function is *used* that needs to mocked, not
# where it is defined.
# see https://docs.python.org/3/library/unittest.mock.html#where-to-patch
"dagster.core.host_representation.handle.sync_get_streaming_external_repositories_grpc"
) as external_repository_mock:
external_repository_mock.side_effect = Exception(
"get_external_repo_failure")
with pytest.warns(
UserWarning,
match=re.escape("get_external_repo_failure")):
workspace = exit_stack.enter_context(
define_out_of_process_workspace(__file__, "get_repo"))
assert not workspace.has_repository_location_handle(
main_repo_location_name())
assert workspace.has_repository_location_error(
main_repo_location_name())
process_context = WorkspaceProcessContext(workspace=workspace,
instance=instance)
assert len(process_context.repository_locations) == 0
workspace.reload_repository_location(main_repo_location_name())
assert workspace.has_repository_location_handle(
main_repo_location_name())
process_context = WorkspaceProcessContext(workspace=workspace,
instance=instance)
assert len(process_context.repository_locations) == 1
def test_handle_cleaup_by_gc_with_dangling_request_reference():
called = {"yup": False}
def call_me():
called["yup"] = True
with instance_for_test() as instance:
with define_out_of_process_workspace(__file__,
"get_repo") as workspace:
# Create a process context
process_context = WorkspaceProcessContext(workspace=workspace,
instance=instance)
process_context.repository_locations[ # pylint: disable=protected-access
0]._handle.cleanup = call_me
assert len(process_context.repository_locations) == 1
assert not called["yup"]
# The request context maintains a reference to the location handle through the
# repository location
request_context = ( # pylint: disable=unused-variable
process_context.create_request_context())
# Even though we reload, verify the handle isn't cleaned up
process_context.reload_repository_location("test_location")
gc.collect()
assert not called["yup"]
# Free reference, make sure handle is cleaned up
request_context = None
gc.collect()
assert called["yup"]
def create_app_from_workspace(workspace: Workspace,
instance: DagsterInstance,
path_prefix: str = ""):
check.inst_param(workspace, "workspace", Workspace)
check.inst_param(instance, "instance", DagsterInstance)
check.str_param(path_prefix, "path_prefix")
if path_prefix:
if not path_prefix.startswith("/"):
raise Exception(
f'The path prefix should begin with a leading "/": got {path_prefix}'
)
if path_prefix.endswith("/"):
raise Exception(
f'The path prefix should not include a trailing "/": got {path_prefix}'
)
warn_if_compute_logs_disabled()
print("Loading repository...") # pylint: disable=print-call
context = WorkspaceProcessContext(instance=instance,
workspace=workspace,
version=__version__)
log_workspace_stats(instance, context)
schema = create_schema()
return instantiate_app_with_views(context, schema, path_prefix)
def test_reload_on_request_context_2():
# This is the similar to the test `test_reload_on_request_context`,
# but calls reload from the request_context instead of on the process_context
with instance_for_test() as instance:
with define_out_of_process_workspace(__file__,
"get_repo") as workspace:
# Create a process context
process_context = WorkspaceProcessContext(workspace=workspace,
instance=instance)
assert len(process_context.repository_locations) == 1
# Save the repository name
repository_location = process_context.repository_locations[0]
repo = list(repository_location.get_repositories().values())[0]
repo_name = repo.name
# Create a request context from the process context
request_context = process_context.create_request_context()
# Reload the location from the request context
new_request_context = request_context.reload_repository_location(
repository_location.name)
# Save the repository name from the:
# - Old request context
# - New request context
# - Process context
repository_location = process_context.repository_locations[0]
repo = list(repository_location.get_repositories().values())[0]
new_repo_name_process_context = repo.name
repository_location = new_request_context.repository_locations[0]
repo = list(repository_location.get_repositories().values())[0]
new_request_context_repo_name = repo.name
repository_location = request_context.repository_locations[0]
repo = list(repository_location.get_repositories().values())[0]
request_context_repo_name = repo.name
assert repo_name == request_context_repo_name
assert request_context_repo_name != new_request_context_repo_name
assert new_repo_name_process_context == new_request_context_repo_name
def test_reload_on_process_context():
with instance_for_test() as instance:
with define_out_of_process_workspace(__file__, "get_repo") as workspace:
# Create a process context
process_context = WorkspaceProcessContext(workspace=workspace, instance=instance)
assert len(process_context.repository_locations) == 1
# Save the repository name
repository_location = process_context.repository_locations[0]
repo = list(repository_location.get_repositories().values())[0]
repo_name = repo.name
# Reload the location and save the new repository name
process_context.reload_repository_location(repository_location.name)
repository_location = process_context.repository_locations[0]
repo = list(repository_location.get_repositories().values())[0]
new_repo_name = repo.name
# Check that the repository has changed
assert repo_name != new_repo_name
def test_handle_cleaup_by_gc_without_request_context():
called = {"yup": False}
def call_me():
called["yup"] = True
with instance_for_test() as instance:
with define_out_of_process_workspace(__file__, "get_repo") as workspace:
# Create a process context
process_context = WorkspaceProcessContext(workspace=workspace, instance=instance)
assert len(process_context.repository_locations) == 1
process_context.repository_locations[ # pylint: disable=protected-access
0
]._handle.cleanup = call_me
# Reload the location from the request context
assert not called["yup"]
process_context.reload_repository_location("test_location")
# There are no more references to the location, so it should be GC'd
gc.collect()
assert called["yup"]
def test_log_workspace_stats(caplog):
with instance_for_test() as instance:
with load_workspace_from_yaml_paths([
file_relative_path(__file__,
"./multi_env_telemetry_workspace.yaml")
]) as workspace:
context = WorkspaceProcessContext(instance=instance,
workspace=workspace)
log_workspace_stats(instance, context)
for record in caplog.records:
message = json.loads(record.getMessage())
assert message.get("action") == UPDATE_REPO_STATS
assert set(message.keys()) == EXPECTED_KEYS
assert len(caplog.records) == 2
def execute_query(workspace,
query,
variables=None,
use_sync_executor=False,
instance=None):
check.inst_param(workspace, "workspace", Workspace)
check.str_param(query, "query")
check.opt_dict_param(variables, "variables")
instance = (check.inst_param(instance, "instance", DagsterInstance)
if instance else DagsterInstance.get())
check.bool_param(use_sync_executor, "use_sync_executor")
query = query.strip("'\" \n\t")
context = WorkspaceProcessContext(
workspace=workspace, instance=instance,
version=__version__).create_request_context()
executor = SyncExecutor() if use_sync_executor else GeventExecutor()
result = graphql(
request_string=query,
schema=create_schema(),
context_value=context,
variable_values=variables,
executor=executor,
)
result_dict = result.to_dict()
# Here we detect if this is in fact an error response
# If so, we iterate over the result_dict and the original result
# which contains a GraphQLError. If that GraphQL error contains
# an original_error property (which is the exception the resolver
# has thrown, typically) we serialize the stack trace of that exception
# in the 'stack_trace' property of each error to ease debugging
if "errors" in result_dict:
check.invariant(len(result_dict["errors"]) == len(result.errors))
for python_error, error_dict in zip(result.errors,
result_dict["errors"]):
if hasattr(python_error,
"original_error") and python_error.original_error:
error_dict["stack_trace"] = get_stack_trace_array(
python_error.original_error)
return result_dict