def _mgr_fn(recon_repo):
"""Goes out of process but same process as host process"""
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
with Workspace(
[
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "setup.py"),
attribute="test_repo",
),
location_name="test",
),
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "setup.py"),
attribute="empty_repo",
),
location_name="empty_repo",
),
]
) as workspace:
yield workspace
def dagit_debug_command(input_files, port):
debug_payloads = []
for input_file in input_files:
click.echo("Loading {} ...".format(input_file))
with GzipFile(input_file, "rb") as file:
blob = file.read().decode()
debug_payload = deserialize_json_to_dagster_namedtuple(blob)
check.invariant(isinstance(debug_payload, DebugRunPayload))
click.echo(
"\trun_id: {} \n\tdagster version: {}".format(
debug_payload.pipeline_run.run_id, debug_payload.version
)
)
debug_payloads.append(debug_payload)
instance = DagsterInstance.ephemeral(preload=debug_payloads)
host_dagit_ui_with_workspace(
workspace=Workspace([]),
instance=instance,
port=port,
port_lookup=True,
host=DEFAULT_DAGIT_HOST,
path_prefix="",
)
def _mgr_fn(recon_repo):
"""Goes out of process but same process as host process"""
check.inst_param(recon_repo, "recon_repo",
ReconstructableRepository)
empty_repo = ReconstructableRepository.from_legacy_repository_yaml(
file_relative_path(__file__, "empty_repo.yaml"))
with Workspace([
RepositoryLocationHandle.create_python_env_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(
__file__, "setup.py"),
attribute="test_repo",
),
location_name="test",
),
RepositoryLocationHandle.create_in_process_location(
empty_repo.pointer),
RepositoryLocationHandle.create_python_env_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(
__file__, "setup.py"),
attribute="empty_repo",
),
location_name="empty_repo",
),
]) as workspace:
yield workspace
def _mgr_fn(recon_repo):
check.inst_param(recon_repo, "recon_repo",
ReconstructableRepository)
with Workspace([
RepositoryLocationHandle.create_in_process_location(
recon_repo.pointer)
]) as workspace:
yield workspace
def define_out_of_process_workspace(python_file, fn_name):
return Workspace(
PythonFileTarget(
python_file=python_file,
attribute=fn_name,
working_directory=None,
location_name=main_repo_location_name(),
))
def define_test_snapshot_context():
return DagsterGraphQLContext(
instance=DagsterInstance.ephemeral(),
workspace=Workspace([
RepositoryLocationHandle.create_in_process_location(
create_main_recon_repo().pointer)
]),
)
def define_context_for_file(python_file, fn_name, instance):
check.inst_param(instance, "instance", DagsterInstance)
return DagsterGraphQLContext(
workspace=Workspace([
RepositoryLocationHandle.create_in_process_location(
CodePointer.from_python_file(python_file, fn_name, None))
]),
instance=instance,
)
def define_context_for_repository_yaml(path, instance):
check.inst_param(instance, "instance", DagsterInstance)
return DagsterGraphQLContext(
workspace=Workspace([
RepositoryLocationHandle.create_in_process_location(
ReconstructableRepository.from_legacy_repository_yaml(
path).pointer)
]),
instance=instance,
)
def _mgr_fn(recon_repo):
"""Goes out of process but same process as host process"""
check.inst_param(recon_repo, "recon_repo",
ReconstructableRepository)
with Workspace(
WorkspaceFileTarget(paths=[
file_relative_path(__file__, "multi_location.yaml")
])) as workspace:
yield workspace
def define_in_process_context(python_file, fn_name, instance):
check.inst_param(instance, "instance", DagsterInstance)
return WorkspaceProcessContext(
workspace=Workspace([
InProcessRepositoryLocationOrigin(
ReconstructableRepository.for_file(python_file, fn_name))
]),
instance=instance,
).create_request_context()
def define_out_of_process_workspace(python_file, fn_name):
return Workspace([
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=python_file,
attribute=fn_name,
),
location_name=main_repo_location_name(),
)
])
def _mgr_fn(recon_repo):
"""Goes out of process but same process as host process"""
check.inst_param(recon_repo, "recon_repo",
ReconstructableRepository)
with Workspace(
PythonFileTarget(
python_file=file_relative_path(__file__, "setup.py"),
attribute="test_dict_repo",
working_directory=None,
location_name="test",
)) as workspace:
yield workspace
def _mgr_fn(recon_repo):
"""Goes out of process via grpc"""
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
loadable_target_origin = recon_repo.get_origin().loadable_target_origin
with Workspace(
[
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=loadable_target_origin, location_name="test",
)
]
) as workspace:
yield workspace
def _mgr_fn(recon_repo):
"""Goes out of process via grpc"""
check.inst_param(recon_repo, "recon_repo",
ReconstructableRepository)
loadable_target_origin = LoadableTargetOrigin.from_python_origin(
recon_repo.get_origin())
with Workspace([
RepositoryLocationHandle.
create_process_bound_grpc_server_location(
loadable_target_origin=loadable_target_origin,
location_name="test",
)
]) as workspace:
yield workspace
def _mgr_fn(recon_repo):
"""Goes out of process but same process as host process"""
check.inst_param(recon_repo, "recon_repo",
ReconstructableRepository)
# this is "ok" because we know the test host process containers the user code
loadable_target_origin = LoadableTargetOrigin.from_python_origin(
recon_repo.get_origin())
with Workspace([
RepositoryLocationHandle.create_python_env_location(
loadable_target_origin=loadable_target_origin,
location_name="test",
)
]) as workspace:
yield workspace
def define_out_of_process_context(python_file, fn_name, instance):
check.inst_param(instance, "instance", DagsterInstance)
return DagsterGraphQLContext(
workspace=Workspace([
RepositoryLocationHandle.create_python_env_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=python_file,
attribute=fn_name,
),
location_name="test_location",
)
]),
instance=instance,
)
def define_out_of_process_context(python_file, fn_name, instance):
check.inst_param(instance, "instance", DagsterInstance)
with Workspace([
ManagedGrpcPythonEnvRepositoryLocationOrigin(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=python_file,
attribute=fn_name,
),
location_name=main_repo_location_name(),
)
]) as workspace:
yield DagsterGraphQLContext(
workspace=workspace,
instance=instance,
)
def _mgr_fn(recon_repo):
"""Goes out of process via grpc"""
check.inst_param(recon_repo, "recon_repo",
ReconstructableRepository)
loadable_target_origin = recon_repo.get_python_origin(
).loadable_target_origin
with Workspace((PythonFileTarget(
python_file=loadable_target_origin.python_file,
attribute=loadable_target_origin.attribute,
working_directory=loadable_target_origin.working_directory,
location_name="test",
) if loadable_target_origin.python_file else ModuleTarget(
module_name=loadable_target_origin.module_name,
attribute=loadable_target_origin.attribute,
location_name="test",
))) as workspace:
yield workspace
def _mgr_fn(recon_repo):
check.inst_param(recon_repo, "recon_repo",
ReconstructableRepository)
loadable_target_origin = recon_repo.get_python_origin(
).loadable_target_origin
server_process = GrpcServerProcess(
loadable_target_origin=loadable_target_origin)
try:
with server_process.create_ephemeral_client() as api_client:
with Workspace(
GrpcServerTarget(
port=api_client.port,
socket=api_client.socket,
host=api_client.host,
location_name="test",
)) as workspace:
yield workspace
finally:
server_process.wait()
def test_execute_hammer_through_dagit():
recon_repo = ReconstructableRepository.for_file(
file_relative_path(
__file__, "../../../dagster-test/dagster_test/toys/hammer.py"),
"hammer_pipeline",
)
instance = DagsterInstance.local_temp()
context = DagsterGraphQLContext(
workspace=Workspace([
RepositoryLocationHandle.create_in_process_location(
recon_repo.pointer)
]),
instance=instance,
)
selector = infer_pipeline_selector(context, "hammer_pipeline")
executor = SyncExecutor()
variables = {
"executionParams": {
"runConfigData": {
"storage": {
"filesystem": {}
},
"execution": {
"dask": {
"config": {
"cluster": {
"local": {}
}
}
}
},
},
"selector": selector,
"mode": "default",
}
}
start_pipeline_result = graphql(
request_string=LAUNCH_PIPELINE_EXECUTION_MUTATION,
schema=create_schema(),
context=context,
variables=variables,
executor=executor,
)
if start_pipeline_result.errors:
raise Exception("{}".format(start_pipeline_result.errors))
run_id = start_pipeline_result.data["launchPipelineExecution"]["run"][
"runId"]
context.drain_outstanding_executions()
subscription = execute_dagster_graphql(context,
SUBSCRIPTION_QUERY,
variables={"runId": run_id})
subscribe_results = []
subscription.subscribe(subscribe_results.append)
messages = [
x["__typename"]
for x in subscribe_results[0].data["pipelineRunLogs"]["messages"]
]
assert "PipelineStartEvent" in messages
assert "PipelineSuccessEvent" in messages
def _mgr_fn(recon_repo):
check.inst_param(recon_repo, "recon_repo", ReconstructableRepository)
with Workspace([InProcessRepositoryLocationOrigin(recon_repo)]) as workspace:
yield workspace
