def dagit_debug_command(input_files, port, asgi):
debug_payloads = []
for input_file in input_files:
click.echo("Loading {} ...".format(input_file))
with GzipFile(input_file, "rb") as file:
blob = file.read().decode("utf-8")
debug_payload = deserialize_json_to_dagster_namedtuple(blob)
check.invariant(isinstance(debug_payload, DebugRunPayload))
click.echo("\trun_id: {} \n\tdagster version: {}".format(
debug_payload.pipeline_run.run_id, debug_payload.version))
debug_payloads.append(debug_payload)
instance = DagsterInstance.ephemeral(preload=debug_payloads)
if asgi:
uvicorn.run(
DagitWebserver(
WorkspaceProcessContext(
instance, None,
version=__version__)).create_asgi_app(debug=True),
port=port,
)
else:
host_dagit_ui_with_workspace_process_context(
workspace_process_context=WorkspaceProcessContext(
instance, None, version=__version__),
port=port,
port_lookup=True,
host=DEFAULT_DAGIT_HOST,
path_prefix="",
)
def _mgr_fn(recon_repo, instance, read_only):
"""Goes out of process via grpc"""
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
loadable_target_origin = recon_repo.get_python_origin().loadable_target_origin
with WorkspaceProcessContext(
instance,
(
PythonFileTarget(
python_file=loadable_target_origin.python_file,
attribute=loadable_target_origin.attribute,
working_directory=loadable_target_origin.working_directory,
location_name="test",
)
if loadable_target_origin.python_file
else ModuleTarget(
module_name=loadable_target_origin.module_name,
attribute=loadable_target_origin.attribute,
location_name="test",
)
),
version="",
read_only=read_only,
) as workspace_process_context:
yield workspace_process_context
def get_deployed_grpc_server_workspace(instance):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
attribute="nope",
python_file=file_relative_path(__file__,
"test_default_run_launcher.py"),
)
server_process = GrpcServerProcess(
loadable_target_origin=loadable_target_origin)
try:
with server_process.create_ephemeral_client(
): # shuts down when leaves this context
with WorkspaceProcessContext(
instance,
GrpcServerTarget(
host="localhost",
socket=server_process.socket,
port=server_process.port,
location_name="test",
),
) as workspace_process_context:
yield workspace_process_context.create_request_context()
finally:
server_process.wait()
def get_workspace_process_context_from_kwargs(
instance: DagsterInstance, version: str, read_only: bool, kwargs
):
from dagster.core.workspace import WorkspaceProcessContext
return WorkspaceProcessContext(
instance, created_workspace_load_target(kwargs), version=version, read_only=read_only
)
def _mgr_fn(instance, read_only):
"""Goes out of process but same process as host process"""
with WorkspaceProcessContext(
instance,
WorkspaceFileTarget(paths=[
file_relative_path(__file__, "multi_location.yaml")
]),
version="",
read_only=read_only,
) as workspace:
yield workspace
def define_out_of_process_workspace(python_file, fn_name, instance):
return WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=python_file,
attribute=fn_name,
working_directory=None,
location_name=main_repo_location_name(),
),
version="",
)
def foo_example_workspace_fixture(instance):
with WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=file_relative_path(__file__, "repo.py"),
attribute=None,
working_directory=os.path.dirname(__file__),
location_name=None,
),
) as workspace_process_context:
yield workspace_process_context.create_request_context()
def get_managed_grpc_server_workspace(instance):
with WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=file_relative_path(__file__, "test_default_run_launcher.py"),
attribute="nope",
working_directory=None,
location_name="test",
),
) as workspace_process_context:
yield workspace_process_context.create_request_context()
def _mgr_fn(recon_repo, instance, read_only):
"""Goes out of process but same process as host process"""
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
with WorkspaceProcessContext(
instance,
WorkspaceFileTarget(paths=[file_relative_path(__file__, "multi_location.yaml")]),
version="",
read_only=read_only,
) as workspace:
yield workspace
def _mgr_fn(instance, read_only):
"""Goes out of process but same process as host process"""
with WorkspaceProcessContext(
instance,
PythonFileTarget(
python_file=file_relative_path(__file__, "setup.py"),
attribute="test_dict_repo",
working_directory=None,
location_name="test",
),
version="",
read_only=read_only,
) as workspace:
yield workspace
def _mgr_fn(instance, read_only):
server_process = GrpcServerProcess(
loadable_target_origin=get_main_loadable_target_origin())
try:
with server_process.create_ephemeral_client() as api_client:
with WorkspaceProcessContext(
instance,
GrpcServerTarget(
port=api_client.port,
socket=api_client.socket,
host=api_client.host,
location_name="test",
),
version="",
read_only=read_only,
) as workspace:
yield workspace
finally:
server_process.wait()
def _mgr_fn(instance, read_only):
"""Goes out of process via grpc"""
loadable_target_origin = get_main_loadable_target_origin()
with WorkspaceProcessContext(
instance,
(PythonFileTarget(
python_file=loadable_target_origin.python_file,
attribute=loadable_target_origin.attribute,
working_directory=loadable_target_origin.working_directory,
location_name="test",
) if loadable_target_origin.python_file else ModuleTarget(
module_name=loadable_target_origin.module_name,
attribute=loadable_target_origin.attribute,
working_directory=loadable_target_origin.working_directory,
location_name="test",
)),
version="",
read_only=read_only,
) as workspace_process_context:
yield workspace_process_context
def workspace_process_context_fixture(instance):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "test_custom_repository_data.py"),
)
server_process = GrpcServerProcess(loadable_target_origin=loadable_target_origin)
try:
with server_process.create_ephemeral_client(): # shuts down when leaves this context
with WorkspaceProcessContext(
instance,
GrpcServerTarget(
host="localhost",
socket=server_process.socket,
port=server_process.port,
location_name="test",
),
) as workspace_process_context:
yield workspace_process_context
finally:
server_process.wait()
def _mgr_fn(recon_repo, instance, read_only):
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
loadable_target_origin = recon_repo.get_python_origin().loadable_target_origin
server_process = GrpcServerProcess(loadable_target_origin=loadable_target_origin)
try:
with server_process.create_ephemeral_client() as api_client:
with WorkspaceProcessContext(
instance,
GrpcServerTarget(
port=api_client.port,
socket=api_client.socket,
host=api_client.host,
location_name="test",
),
version="",
read_only=read_only,
) as workspace:
yield workspace
finally:
server_process.wait()